├── setup.cfg ├── protoparser ├── __init__.py └── parser.py ├── setup.py ├── test └── test.py ├── README.md └── LICENSE /setup.cfg: -------------------------------------------------------------------------------- 1 | [egg_info] 2 | tag_build= 3 | -------------------------------------------------------------------------------- /protoparser/__init__.py: -------------------------------------------------------------------------------- 1 | from protoparser.parser import parse, parse_from_file, serialize2json_from_file 2 | from protoparser.parser import serialize2json 3 | 4 | __version__ = "1.6.3" 5 | __all__ = [parse, parse_from_file, serialize2json, serialize2json_from_file] 6 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | with open("README.md", "r") as fh: 4 | long_description = fh.read() 5 | 6 | setuptools.setup( 7 | name="proto-parser", 8 | version="1.6.3", 9 | author="xiaochun.liu", 10 | author_email="liuxiaochun@apache.org", 11 | description="A package for parsing proto files", 12 | long_description_content_type="text/markdown", 13 | url="https://github.com/khadgarmage/protoparser", 14 | packages=setuptools.find_packages(), 15 | classifiers=[ 16 | "Programming Language :: Python :: 3", 17 | "License :: OSI Approved :: Apache Software License", 18 | "Operating System :: OS Independent", 19 | "Topic :: Software Development :: Libraries :: Python Modules", 20 | ], 21 | install_requires=[ 22 | 'lark-parser>=0.8.6', 23 | 'numpy>=1.14.0' 24 | ], 25 | ) 26 | -------------------------------------------------------------------------------- /test/test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | import protoparser 20 | 21 | content = '''syntax = "proto3"; 22 | package service; 23 | option go_package = "bitbucket.org/funplus/sandwichgmt/backend/pkg/gen/service"; 24 | import "msg/job_file.proto"; 25 | import "google/api/annotations.proto"; 26 | 27 | // 28 | 29 | message MessageItem { 30 | string Title = 1; 31 | string Content = 2;//你好啊 32 | } 33 | 34 | //背包类型 35 | enum BagType { 36 | Other = 0; 37 | Bag = 1; 38 | Store = 2; 39 | } 40 | 41 | enum PlayerType { 42 | //@ignore 43 | NORMAL = 0; 44 | //cheater 45 | CHEATER = 1; 46 | //tester 47 | TESTER = 2; 48 | //deleted player 49 | DELETE = 3; 50 | } 51 | 52 | //@entry 53 | //@schema 54 | message Player { 55 | //player id 56 | uint64 PlayerId = 1;//Player ID 57 | //Name 58 | string Name = 2; 59 | //@max=1000 60 | int32 Level = 3; 61 | int32 Coins = 4; 62 | //@fmt=date 63 | //@desc=Player's birthday 64 | string Birthday = 5; 65 | //@required 66 | PlayerType Type = 6;///YYY 67 | // @title=App version history 68 | repeated string AppVerHistory = 7; 69 | repeated MessageItem MessageBox = 8; 70 | message StoreItem { 71 | uint32 Num = 1; 72 | //@title 73 | string From = 2; 74 | } 75 | //@ title =Warehouse 76 | map Storage = 9; 77 | //fmt =email 78 | string Email = 10; 79 | //@pattern=^(https?|ftp|file)://[-A-Za-z0-9+&@#/%?=~_|!:,.;]+[-A-Za-z0-9+&@#/%=~_|]$ 80 | string HomePage = 11; // 81 | enum InnerType { 82 | TEST = 0; 83 | OK = 1; 84 | } 85 | repeated string _tags_ = 19;//你好啊 86 | } 87 | 88 | message MissionTeamSaveResponse { 89 | map MissionTeam = 1; //你好啊 90 | } 91 | 92 | //fdsafsadfdsafsa 93 | service JobFileService { 94 | //fdsafdsa 95 | rpc GDriveFileList (msg.GDriveFileListReq) returns(msg.FileListRep) { 96 | option (google.api.http) = { 97 | post: "/api/files/gdrive" 98 | body: "*" 99 | }; 100 | } 101 | //fdsafdsa 102 | rpc GDriveFileListx (msg.GDriveFileListReqx) returns(msg.FileListRepx) { 103 | option (google.api.http) = { 104 | post: "/api/files/gdrivex" 105 | body: "*" 106 | }; 107 | } 108 | } 109 | 110 | message Daadd { 111 | map MissionTeam = 1; // ewrew 112 | } 113 | ''' 114 | # data = protoparser.parse(content) 115 | # for i in data.messages: 116 | # message = data.messages[i] 117 | print(protoparser.serialize2json(content)) 118 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Protoparser 2 | A package for parsing proto3 files 3 | ## Introduction 4 | The purpose of this package is to parse the .proto file (version 3) into a Python data structure. 5 | We use it for code generation or other operations. 6 | ## How to Use 7 | ``` 8 | pip install proto-parser 9 | ``` 10 | Output format is as following: 11 | ```json 12 | 13 | { 14 | "messages": { 15 | "MessageItem": { 16 | "comment": { 17 | "content": "", 18 | "tags": {} 19 | }, 20 | "name": "MessageItem", 21 | "fields": [ 22 | { 23 | "comment": { 24 | "content": "", 25 | "tags": {} 26 | }, 27 | "type": "string", 28 | "key_type": "string", 29 | "val_type": "string", 30 | "name": "Title", 31 | "number": 1 32 | } 33 | ], 34 | "messages": {}, 35 | "enums": {} 36 | }, 37 | "Player": { 38 | "comment": { 39 | "content": "//@entry\n//@schema\n", 40 | "tags": { 41 | "entry": true, 42 | "schema": true 43 | } 44 | }, 45 | "name": "Player", 46 | "fields": [ 47 | { 48 | "comment": { 49 | "content": "//@fmt=date\n//@desc=Player's birthday\n", 50 | "tags": { 51 | "fmt": "date", 52 | "desc": "Player's birthday" 53 | } 54 | }, 55 | "type": "string", 56 | "key_type": "string", 57 | "val_type": "string", 58 | "name": "Birthday", 59 | "number": 5 60 | }, 61 | { 62 | "comment": { 63 | "content": "//@required\n", 64 | "tags": { 65 | "required": true 66 | } 67 | }, 68 | "type": "PlayerType", 69 | "key_type": "PlayerType", 70 | "val_type": "PlayerType", 71 | "name": "Type", 72 | "number": 6 73 | }, 74 | { 75 | "comment": { 76 | "content": "// @title App version history\n", 77 | "tags": {} 78 | }, 79 | "type": "repeated", 80 | "key_type": "string", 81 | "val_type": "string", 82 | "name": "AppVerHistory", 83 | "number": 7 84 | }, 85 | { 86 | "comment": { 87 | "content": "", 88 | "tags": {} 89 | }, 90 | "type": "repeated", 91 | "key_type": "MessageItem", 92 | "val_type": "MessageItem", 93 | "name": "MessageBox", 94 | "number": 8 95 | }, 96 | { 97 | "comment": { 98 | "content": "//@ title =Warehouse\n", 99 | "tags": { 100 | "title": "Warehouse" 101 | } 102 | }, 103 | "type": "map", 104 | "key_type": "uint64", 105 | "val_type": "StoreItem", 106 | "name": "Storage", 107 | "number": 9 108 | }, 109 | { 110 | "comment": { 111 | "content": "//@pattern=^(https?|ftp|file)://[-A-Za-z0-9+&@#/%?=~_|!:,.;]+[-A-Za-z0-9+&@#/%=~_|]$\n", 112 | "tags": { 113 | "pattern": "^(https?|ftp|file)://[-A-Za-z0-9+&", 114 | "#/%?": "~_|!:,.;]+[-A-Za-z0-9+&", 115 | "#/%": "~_|]$" 116 | } 117 | }, 118 | "type": "string", 119 | "key_type": "string", 120 | "val_type": "string", 121 | "name": "HomePage", 122 | "number": 11 123 | } 124 | ], 125 | "messages": { 126 | "StoreItem": { 127 | "comment": { 128 | "content": "", 129 | "tags": {} 130 | }, 131 | "name": "StoreItem", 132 | "fields": [ 133 | { 134 | "comment": { 135 | "content": "", 136 | "tags": {} 137 | }, 138 | "type": "uint32", 139 | "key_type": "uint32", 140 | "val_type": "uint32", 141 | "name": "Num", 142 | "number": 1 143 | } 144 | ], 145 | "messages": {}, 146 | "enums": {} 147 | } 148 | }, 149 | "enums": { 150 | "InnerType": { 151 | "comment": { 152 | "content": "", 153 | "tags": {} 154 | }, 155 | "name": "InnerType", 156 | "fields": [ 157 | { 158 | "comment": { 159 | "content": "", 160 | "tags": {} 161 | }, 162 | "type": "enum", 163 | "key_type": "enum", 164 | "val_type": "enum", 165 | "name": "TEST", 166 | "number": "0" 167 | } 168 | ] 169 | } 170 | } 171 | } 172 | }, 173 | "enums": { 174 | "PlayerType": { 175 | "comment": { 176 | "content": "", 177 | "tags": {} 178 | }, 179 | "name": "PlayerType", 180 | "fields": [ 181 | { 182 | "comment": { 183 | "content": "//normal player\n", 184 | "tags": {} 185 | }, 186 | "type": "enum", 187 | "key_type": "enum", 188 | "val_type": "enum", 189 | "name": "NORMAL", 190 | "number": "0" 191 | }, 192 | { 193 | "comment": { 194 | "content": "//cheater\n", 195 | "tags": {} 196 | }, 197 | "type": "enum", 198 | "key_type": "enum", 199 | "val_type": "enum", 200 | "name": "CHEATER", 201 | "number": "1" 202 | } 203 | ] 204 | } 205 | }, 206 | "services": { 207 | "JobFileService": { 208 | "name": "JobFileService", 209 | "functions": [ 210 | { 211 | "name": "GDriveFileList", 212 | "in_type": "msg.GDriveFileListReq", 213 | "out_type": "msg.FileListRep", 214 | "uri": "/api/files/gdrive" 215 | } 216 | ] 217 | } 218 | } 219 | } 220 | ``` 221 | ## Bug Reports and Patches 222 | If you think you have found a bug, please visit the Protoparser Github page at https://github.com/khadgarmage/protoparser 223 | to report an issue, or fix it to push a pull request, thanks. 224 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /protoparser/parser.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | from lark import Lark, Transformer, Tree, Token 19 | from collections import namedtuple 20 | import typing 21 | import json 22 | 23 | BNF = r''' 24 | OCTALDIGIT: "0..7" 25 | IDENT: ( "_" )* LETTER ( LETTER | DECIMALDIGIT | "_" )* 26 | FULLIDENT: IDENT ( "." IDENT )* 27 | MESSAGENAME: IDENT 28 | ENUMNAME: IDENT 29 | FIELDNAME: IDENT 30 | ONEOFNAME: IDENT 31 | MAPNAME: IDENT 32 | SERVICENAME: IDENT 33 | TAGNAME: IDENT 34 | TAGVALUE: IDENT 35 | RPCNAME: IDENT 36 | MESSAGETYPE: [ "." ] ( IDENT "." )* MESSAGENAME 37 | ENUMTYPE: [ "." ] ( IDENT "." )* ENUMNAME 38 | 39 | INTLIT : DECIMALLIT | OCTALLIT | HEXLIT 40 | DECIMALLIT: ( "1".."9" ) ( DECIMALDIGIT )* 41 | OCTALLIT : "0" ( OCTALDIGIT )* 42 | HEXLIT : "0" ( "x" | "X" ) HEXDIGIT ( HEXDIGIT )* 43 | 44 | FLOATLIT: ( DECIMALS "." [ DECIMALS ] [ EXPONENT ] | DECIMALS EXPONENT | "."DECIMALS [ EXPONENT ] ) | "inf" | "nan" 45 | DECIMALS : DECIMALDIGIT ( DECIMALDIGIT )* 46 | EXPONENT : ( "e" | "E" ) [ "+" | "-" ] DECIMALS 47 | 48 | BOOLLIT: "true" | "false" 49 | 50 | STRLIT: ( "'" ( CHARVALUE )* "'" ) | ( "\"" ( CHARVALUE )* "\"" ) 51 | CHARVALUE: HEXESCAPE | OCTESCAPE | CHARESCAPE | /[^\0\n\\]/ 52 | HEXESCAPE: "\\" ( "x" | "X" ) HEXDIGIT HEXDIGIT 53 | OCTESCAPE: "\\" OCTALDIGIT OCTALDIGIT OCTALDIGIT 54 | CHARESCAPE: "\\" ( "a" | "b" | "f" | "n" | "r" | "t" | "v" | "\\" | "'" | "\"" ) 55 | QUOTE: "'" | "\"" 56 | 57 | EMPTYSTATEMENT: ";" 58 | 59 | CONSTANT: FULLIDENT | ( [ "-" | "+" ] INTLIT ) | ( [ "-" | "+" ] FLOATLIT ) | STRLIT | BOOLLIT 60 | 61 | syntax: "syntax" "=" QUOTE "proto3" QUOTE ";" 62 | 63 | import: "import" [ "weak" | "public" ] STRLIT ";" 64 | 65 | package: "package" FULLIDENT ";" 66 | 67 | option: "option" OPTIONNAME "=" CONSTANT ";" 68 | OPTIONNAME: ( IDENT | "(" FULLIDENT ")" ) ( "." IDENT )* 69 | 70 | TYPE: "double" | "float" | "int32" | "int64" | "uint32" | "uint64" | "sint32" | "sint64" | "fixed32" | "fixed64" | "sfixed32" | "sfixed64" | "bool" | "string" | "bytes" | MESSAGETYPE | ENUMTYPE 71 | FIELDNUMBER: INTLIT 72 | 73 | field: [ comments ] TYPE FIELDNAME "=" FIELDNUMBER [ "[" fieldoptions "]" ] TAIL 74 | fieldoptions: fieldoption ( "," fieldoption )* 75 | fieldoption: OPTIONNAME "=" CONSTANT 76 | repeatedfield: [ comments ] "repeated" field 77 | 78 | oneof: "oneof" ONEOFNAME "{" ( oneoffield | EMPTYSTATEMENT )* "}" 79 | oneoffield: TYPE FIELDNAME "=" FIELDNUMBER [ "[" fieldoptions "]" ] ";" 80 | 81 | mapfield: [ comments ] "map" "<" KEYTYPE "," TYPE ">" MAPNAME "=" FIELDNUMBER [ "[" fieldoptions "]" ] TAIL 82 | KEYTYPE: "int32" | "int64" | "uint32" | "uint64" | "sint32" | "sint64" | "fixed32" | "fixed64" | "sfixed32" | "sfixed64" | "bool" | "string" 83 | 84 | reserved: "reserved" ( ranges | fieldnames ) ";" 85 | ranges: range ( "," range )* 86 | range: INTLIT [ "to" ( INTLIT | "max" ) ] 87 | fieldnames: FIELDNAME ( "," FIELDNAME )* 88 | 89 | enum: [ comments ] "enum" ENUMNAME enumbody 90 | enumbody: "{" ( enumfield | EMPTYSTATEMENT )* "}" 91 | enumfield: [ COMMENTS ] IDENT "=" INTLIT [ "[" enumvalueoption ( "," enumvalueoption )* "]" ] TAIL 92 | enumvalueoption: OPTIONNAME "=" CONSTANT 93 | 94 | message: [ comments ] "message" MESSAGENAME messagebody 95 | messagebody: "{" ( repeatedfield | field | enum | message | option | oneof | mapfield | reserved | EMPTYSTATEMENT )* "}" 96 | 97 | googleoption: "option" "(google.api.http)" "=" "{" [ "post:" CONSTANT [ "body:" CONSTANT ] ] "}" ";" 98 | service: [ comments ] "service" SERVICENAME "{" ( option | rpc | EMPTYSTATEMENT )* "}" 99 | rpc: [ comments ] "rpc" RPCNAME "(" [ "stream" ] MESSAGETYPE ")" "returns" "(" [ "stream" ] MESSAGETYPE ")" ( ( "{" ( googleoption | option | EMPTYSTATEMENT )* "}" ) | ";" ) 100 | 101 | proto:[ comments ] syntax ( import | package | option | topleveldef | EMPTYSTATEMENT )* 102 | topleveldef: message | enum | service | comments 103 | 104 | TAIL: ";" [/[\s|\t]/] [ COMMENT ] 105 | COMMENT: "//" /.*/ [ "\n" ] 106 | comments: COMMENT ( COMMENT )* 107 | COMMENTS: COMMENT ( COMMENT )* 108 | 109 | %import common.HEXDIGIT 110 | %import common.DIGIT -> DECIMALDIGIT 111 | %import common.LETTER 112 | %import common.WS 113 | %import common.NEWLINE 114 | %ignore WS 115 | ''' 116 | 117 | Comment = typing.NamedTuple('Comment', [('content', str), ('tags', typing.Dict[str, typing.Any])]) 118 | Field = typing.NamedTuple('Field', [('comment', 'Comment'), ('type', str), ('key_type', str), ('val_type', str), ('name', str), ('number', int)]) 119 | Enum = typing.NamedTuple('Enum', [('comment', 'Comment'), ('name', str), ('fields', typing.Dict[str, 'Field'])]) 120 | Message = typing.NamedTuple('Message', [('comment', 'Comment'), ('name', str), ('fields', typing.List['Field']), 121 | ('messages', typing.Dict[str, 'Message']), ('enums', typing.Dict[str, 'Enum'])]) 122 | Service = typing.NamedTuple('Service', [('name', str), ('functions', typing.Dict[str, 'RpcFunc'])]) 123 | RpcFunc = typing.NamedTuple('RpcFunc', [('name', str), ('in_type', str), ('out_type', str), ('uri', str)]) 124 | ProtoFile = typing.NamedTuple('ProtoFile', 125 | [('messages', typing.Dict[str, 'Message']), ('enums', typing.Dict[str, 'Enum']), 126 | ('services', typing.Dict[str, 'Service']), ('imports', typing.List[str]), 127 | ('options', typing.Dict[str, str]), ('package', str)]) 128 | 129 | 130 | class ProtoTransformer(Transformer): 131 | '''Converts syntax tree token into more easily usable namedtuple objects''' 132 | 133 | def message(self, tokens): 134 | '''Returns a Message namedtuple''' 135 | comment = Comment("", {}) 136 | if len(tokens) < 3: 137 | name_token, body = tokens 138 | else: 139 | comment, name_token, body = tokens 140 | return Message(comment, name_token.value, *body) 141 | 142 | def messagebody(self, items): 143 | '''Returns a tuple of message body namedtuples''' 144 | messages = {} 145 | enums = {} 146 | fields = [] 147 | for item in items: 148 | if isinstance(item, Message): 149 | messages[item.name] = item 150 | elif isinstance(item, Enum): 151 | enums[item.name] = item 152 | elif isinstance(item, Field): 153 | fields.append(item) 154 | return fields, messages, enums 155 | 156 | def field(self, tokens): 157 | '''Returns a Field namedtuple''' 158 | comment = Comment("", {}) 159 | type = Token("TYPE", "") 160 | fieldname = Token("FIELDNAME", "") 161 | fieldnumber = Token("FIELDNUMBER", "") 162 | for token in tokens: 163 | if isinstance(token, Comment): 164 | comment = token 165 | elif isinstance(token, Token): 166 | if token.type == "TYPE": 167 | type = token 168 | elif token.type == "FIELDNAME": 169 | fieldname = token 170 | elif token.type == "FIELDNUMBER": 171 | fieldnumber = token 172 | elif token.type == "COMMENT": 173 | comment = Comment(token.value, {}) 174 | return Field(comment, type.value, type.value, type.value, fieldname.value, int(fieldnumber.value)) 175 | 176 | def repeatedfield(self, tokens): 177 | '''Returns a Field namedtuple''' 178 | comment = Comment("", {}) 179 | if len(tokens) < 2: 180 | field = tokens[0] 181 | else: 182 | comment, field = tuple(tokens) 183 | return Field(comment, 'repeated', field.type, field.type, field.name, field.number) 184 | 185 | def mapfield(self, tokens): 186 | '''Returns a Field namedtuple''' 187 | comment = Comment("", {}) 188 | val_type = Token("TYPE", "") 189 | key_type = Token("KEYTYPE", "") 190 | fieldname = Token("MAPNAME", "") 191 | fieldnumber = Token("FIELDNUMBER", "") 192 | for token in tokens: 193 | if isinstance(token, Comment): 194 | comment = token 195 | elif isinstance(token, Token): 196 | if token.type == "TYPE": 197 | val_type = token 198 | elif token.type == "KEYTYPE": 199 | key_type = token 200 | elif token.type == "MAPNAME": 201 | fieldname = token 202 | elif token.type == "FIELDNUMBER": 203 | fieldnumber = token 204 | elif token.type == "COMMENT": 205 | comment = Comment(token.value, {}) 206 | return Field(comment, 'map', key_type.value, val_type.value, fieldname.value, int(fieldnumber.value)) 207 | 208 | def comments(self, tokens): 209 | '''Returns a Tag namedtuple''' 210 | comment = '' 211 | tags = {} 212 | for token in tokens: 213 | comment += token 214 | if token.find('@') < 0: 215 | continue 216 | kvs = token.strip(" /\n").split('@') 217 | for kv in kvs: 218 | kv = kv.strip(" /\n") 219 | if not kv: 220 | continue 221 | tmp = kv.split('=') 222 | key = tmp[0].strip(" /\n").lower() 223 | if key.find(" ") >= 0: 224 | continue 225 | if len(tmp) > 1: 226 | tags[key] = tmp[1].lower() 227 | else: 228 | tags[key] = True 229 | return Comment(comment, tags) 230 | 231 | def enum(self, tokens): 232 | '''Returns an Enum namedtuple''' 233 | comment = Comment("", {}) 234 | if len(tokens) < 3: 235 | name, fields = tokens 236 | else: 237 | comment, name, fields = tokens 238 | return Enum(comment, name.value, fields) 239 | 240 | def enumbody(self, tokens): 241 | '''Returns a sequence of enum identifiers''' 242 | enumitems = [] 243 | for tree in tokens: 244 | if tree.data != 'enumfield': 245 | continue 246 | comment = Comment("", {}) 247 | name = Token("IDENT", "") 248 | value = Token("INTLIT", "") 249 | for token in tree.children: 250 | if isinstance(token, Comment): 251 | comment = token 252 | elif isinstance(token, Token): 253 | if token.type == "IDENT": 254 | name = token 255 | elif token.type == "INTLIT": 256 | value = token 257 | elif token.type == "COMMENTS": 258 | comment = Comment(token.value, {}) 259 | enumitems.append(Field(comment, 'enum', 'enum', 'enum', name.value, value.value)) 260 | return enumitems 261 | 262 | def service(self, tokens): 263 | '''Returns a Service namedtuple''' 264 | functions = [] 265 | name = '' 266 | for i in range(0, len(tokens)): 267 | if not isinstance(tokens[i], Comment): 268 | if isinstance(tokens[i], RpcFunc): 269 | functions.append(tokens[i]) 270 | else: 271 | name = tokens[i].value 272 | return Service(name, functions) 273 | 274 | def rpc(self, tokens): 275 | '''Returns a RpcFunc namedtuple''' 276 | uri = '' 277 | in_type = '' 278 | for token in tokens: 279 | if isinstance(token, Token): 280 | if token.type == "RPCNAME": 281 | name = token 282 | elif token.type == "MESSAGETYPE": 283 | if in_type: 284 | out_type = token 285 | else: 286 | in_type = token 287 | elif not isinstance(token, Comment): 288 | option_token = token 289 | uri = option_token.children[0].value 290 | return RpcFunc(name.value, in_type.value, out_type.value, uri.strip('"')) 291 | 292 | 293 | def _recursive_to_dict(obj): 294 | _dict = {} 295 | 296 | if isinstance(obj, tuple): 297 | node = obj._asdict() 298 | for item in node: 299 | if isinstance(node[item], list): # Process as a list 300 | _dict[item] = [_recursive_to_dict(x) for x in (node[item])] 301 | elif isinstance(node[item], tuple): # Process as a NamedTuple 302 | _dict[item] = _recursive_to_dict(node[item]) 303 | elif isinstance(node[item], dict): 304 | for k in node[item]: 305 | if isinstance(node[item][k], tuple): 306 | node[item][k] = _recursive_to_dict(node[item][k]) 307 | _dict[item] = node[item] 308 | else: # Process as a regular element 309 | _dict[item] = (node[item]) 310 | return _dict 311 | 312 | 313 | def parse_from_file(file: str): 314 | with open(file, 'r') as f: 315 | data = f.read() 316 | if data: 317 | return parse(data) 318 | 319 | 320 | def parse(data: str): 321 | parser = Lark(BNF, start='proto', parser='lalr') 322 | tree = parser.parse(data) 323 | trans_tree = ProtoTransformer().transform(tree) 324 | enums = {} 325 | messages = {} 326 | services = {} 327 | imports = [] 328 | import_tree = trans_tree.find_data('import') 329 | for tree in import_tree: 330 | for child in tree.children: 331 | imports.append(child.value.strip('"')) 332 | options = {} 333 | option_tree = trans_tree.find_data('option') 334 | for tree in option_tree: 335 | options[tree.children[0]] = tree.children[1].strip('"') 336 | 337 | package = '' 338 | package_tree = trans_tree.find_data('package') 339 | for tree in package_tree: 340 | package = tree.children[0] 341 | 342 | top_data = trans_tree.find_data('topleveldef') 343 | for top_level in top_data: 344 | for child in top_level.children: 345 | if isinstance(child, Message): 346 | messages[child.name] = child 347 | if isinstance(child, Enum): 348 | enums[child.name] = child 349 | if isinstance(child, Service): 350 | services[child.name] = child 351 | return ProtoFile(messages, enums, services, imports, options, package) 352 | 353 | 354 | def serialize2json(data): 355 | return json.dumps(_recursive_to_dict(parse(data))) 356 | 357 | 358 | def serialize2json_from_file(file: str): 359 | with open(file, 'r') as f: 360 | data = f.read() 361 | if data: 362 | return json.dumps(_recursive_to_dict(parse(data))) 363 | --------------------------------------------------------------------------------