├── .editorconfig ├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── compatibility.md ├── dprotoc ├── dub.json └── src │ └── app.d ├── dub.json ├── examples ├── proto │ └── person.proto ├── simple.d └── simple_file.d └── import └── dproto ├── attributes.d ├── compat.d ├── dproto.d ├── exception.d ├── imports.d ├── intermediate.d ├── package.d ├── parse.d ├── serialize.d └── unittests.d /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*.{c,h,d,di,dd,json}] 4 | end_of_line = lf 5 | insert_final_newline = true 6 | indent_style = tab 7 | indent_size = 4 8 | trim_trailing_whitespace = true 9 | charset = utf-8 10 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .dub 2 | *.a 3 | *.o 4 | *.sw[po] 5 | __test__library__ 6 | dub.selections.json 7 | dproto_dprotoc 8 | dproto-test-library 9 | *.lib 10 | *.exe 11 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: d 2 | d: 3 | - dmd 4 | - dmd-2.090.0 5 | - dmd-2.089.1 6 | - dmd-2.088.1 7 | - gdc 8 | - ldc 9 | - ldc-1.19.0 10 | - ldc-1.18.0 11 | - ldc-1.17.0 12 | addons: 13 | apt: 14 | packages: 15 | - libatomic1 16 | before_install: 17 | - dub fetch doveralls 18 | script: 19 | - dub test --coverage 20 | after_success: 21 | - dub run doveralls 22 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Boost Software License - Version 1.0 - August 17th, 2003 2 | 3 | Permission is hereby granted, free of charge, to any person or organization 4 | obtaining a copy of the software and accompanying documentation covered by 5 | this license (the "Software") to use, reproduce, display, distribute, 6 | execute, and transmit the Software, and to prepare derivative works of the 7 | Software, and to permit third-parties to whom the Software is furnished to 8 | do so, all subject to the following: 9 | 10 | The copyright notices in the Software and this entire statement, including 11 | the above license grant, this restriction and the following disclaimer, 12 | must be included in all copies of the Software, in whole or in part, and 13 | all derivative works of the Software, unless such copies or derivative 14 | works are solely in the form of machine-executable object code generated by 15 | a source language processor. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT 20 | SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE 21 | FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, 22 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 23 | DEALINGS IN THE SOFTWARE. 24 | 25 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # D Protocol Buffers 2 | 3 | ---- 4 | 5 | [![Build Status](https://travis-ci.org/msoucy/dproto.svg?branch=master)](https://travis-ci.org/msoucy/dproto) 6 | [![Coverage Status](https://coveralls.io/repos/msoucy/dproto/badge.svg?branch=master)](https://coveralls.io/r/msoucy/dproto) 7 | [![DUB](https://img.shields.io/dub/dt/dproto/latest.svg)](http://code.dlang.org/packages/dproto) 8 | [![DUB license](https://img.shields.io/dub/l/dproto.svg)](http://code.dlang.org/packages/dproto) 9 | 10 | Protocol buffers are a language-agnostic way of specifying message structures to allow communication and serialization. 11 | 12 | `dproto` is designed to enable mixing protocol buffer files into your D code at compile time. 13 | 14 | Inspiration and a good portion of the original parser is adapted from [square/protoparser](http://github.com/square/protoparser) 15 | 16 | ---- 17 | 18 | # Options 19 | 20 | `dproto` supports altering behavior via `protobuf` options: 21 | 22 | | Option | Meaning | Example | Default | 23 | |-----------------------|--------------------------------------------------------|----------------------------------------------|---------| 24 | | `dproto_reserved_fmt` | The format for renaming reserved D keywords as fields. | `"%s_"` will convert `version` to `version_` | `"%s_"` | 25 | 26 | ---- 27 | 28 | # Examples 29 | 30 | [Further info](https://developers.google.com/protocol-buffers/docs/overview) 31 | 32 | Examples can be found in `import/dproto/dproto.d` and in `examples/`. 33 | 34 | ## Simple Example 35 | 36 | ```d 37 | import std.stdio; 38 | import dproto.dproto; 39 | 40 | mixin ProtocolBufferFromString!" 41 | message Person { 42 | required string name = 1; 43 | required int32 id = 2; 44 | optional string email = 3; 45 | 46 | enum PhoneType { 47 | MOBILE = 0; 48 | HOME = 1; 49 | WORK = 2; 50 | } 51 | 52 | message PhoneNumber { 53 | required string number = 1; 54 | optional PhoneType type = 2 [default = HOME]; 55 | } 56 | 57 | repeated PhoneNumber phone = 4; 58 | } 59 | "; 60 | 61 | 62 | int main() 63 | { 64 | Person person; 65 | person.name = "John Doe"; 66 | person.id = 1234; 67 | person.email = "jdoe@example.com"; 68 | 69 | ubyte[] serializedObject = person.serialize(); 70 | 71 | Person person2 = Person(serializedObject); 72 | writeln("Name: ", person2.name); 73 | writeln("E-mail: ", person2.email); 74 | return 0; 75 | } 76 | ``` 77 | 78 | ## More Complex Example 79 | 80 | ```d 81 | import dproto.dproto; 82 | 83 | mixin ProtocolBufferFromString!" 84 | enum PhoneType { 85 | MOBILE = 0; 86 | HOME = 0; 87 | WORK = 2; 88 | } 89 | 90 | message Person { 91 | required string name = 1; 92 | required int32 id = 2; 93 | optional string email = 3; 94 | 95 | message PhoneNumber { 96 | required string number = 1; 97 | optional PhoneType type = 2 [default = HOME]; 98 | } 99 | 100 | repeated PhoneNumber phone = 4; 101 | } 102 | 103 | message AddressBook { 104 | repeated Person person = 1; 105 | } 106 | "; 107 | 108 | 109 | int main() 110 | { 111 | Person t; 112 | t.name = "Max Musterman"; 113 | t.id = 3; 114 | t.email = "test@example.com"; 115 | 116 | Person.PhoneNumber pn1; 117 | pn1.number = "0123456789"; 118 | pn1.type = PhoneType.WORK; 119 | 120 | Person.PhoneNumber pn2; 121 | pn2.number = "0123456789"; 122 | 123 | t.phone = [pn1, pn2]; 124 | AddressBook addressbook; 125 | addressbook.person ~= t; 126 | addressbook.person ~= t; 127 | 128 | ubyte[] serializedObject = addressbook.serialize(); 129 | 130 | AddressBook addressbook2 = AddressBook(serializedObject); 131 | assert(addressbook2.person.length == 2); 132 | foreach(t2; addressbook2.person) 133 | { 134 | assert(t2.name == "Max Musterman"); 135 | assert(t2.id == 3); 136 | assert(t2.email == "test@example.com"); 137 | assert(t2.email !is null); 138 | assert(t2.phone[0].number == "0123456789"); 139 | assert(t2.phone[0].type == PhoneType.WORK); 140 | assert(t2.phone[1].number == "0123456789"); 141 | assert(t2.phone[1].type == PhoneType.HOME); 142 | assert(t2.phone[1].type == PhoneType.MOBILE); 143 | assert(t2.phone.length == 2); 144 | } 145 | version(DigitalMars) 146 | { 147 | assert(addressbook2.person[0] == addressbook.person[1]); 148 | } 149 | return 0; 150 | } 151 | ``` 152 | 153 | 154 | ## Services 155 | 156 | Generate interfaces for service definitions. 157 | 158 | ```d 159 | import dproto.dproto; 160 | 161 | mixin ProtocolBufferInterface!" 162 | message ServiceRequest { 163 | string request = 1; 164 | } 165 | message ServiceResponse { 166 | string response = 1; 167 | } 168 | service TestService { 169 | rpc TestMethod (ServiceRequest) returns (ServiceResponse); 170 | } 171 | "; 172 | 173 | class ServiceImplementation : TestService { 174 | ServiceResponse TestMethod(ServiceRequest input) { 175 | ServiceResponse output; 176 | output.response = "received: " ~ input.request; 177 | return output; 178 | } 179 | } 180 | ``` 181 | -------------------------------------------------------------------------------- /compatibility.md: -------------------------------------------------------------------------------- 1 | Compatibility with protobuf 2 | =========================== 3 | 4 | Compatible 5 | ---------- 6 | 7 | * Serialization of the primitives 8 | * Nested structures 9 | * Required values 10 | * Optional values 11 | * Remote procedure calls ("RPC") 12 | 13 | Partially Compatible 14 | -------------------- 15 | 16 | * Import statements (does not support language builtins like `Any`) 17 | 18 | Not Implemented 19 | --------------- 20 | 21 | * Extensions 22 | * Options 23 | * Mapping types 24 | * Reserved fields 25 | -------------------------------------------------------------------------------- /dprotoc/dub.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dprotoc", 3 | "targetType": "executable", 4 | "dependencies": { 5 | "dproto": {"path": ".."} 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /dprotoc/src/app.d: -------------------------------------------------------------------------------- 1 | module dproto.app; 2 | 3 | import std.range; 4 | import std.algorithm; 5 | import std.getopt; 6 | import std.stdio; 7 | import std.string; 8 | import dproto.parse; 9 | import dproto.intermediate; 10 | 11 | auto openFileComplex(string fn, string mode) 12 | { 13 | if (fn == "-") 14 | { 15 | return (mode == "r") ? stdin : stdout; 16 | } 17 | else 18 | { 19 | return File(fn, mode); 20 | } 21 | } 22 | 23 | void main(string[] args) 24 | { 25 | string infile = "-"; 26 | string outfile = "-"; 27 | string fmt = "%d"; 28 | auto helpInformation = getopt( 29 | args, 30 | "out|o", "Output filename (default stdout)", &outfile, 31 | "format|f", "Code generation format", &fmt, 32 | ); 33 | if (helpInformation.helpWanted) 34 | { 35 | defaultGetoptPrinter( 36 | "Protocol Buffer D generator", 37 | helpInformation.options); 38 | return; 39 | } 40 | ProtoPackage pack; 41 | { 42 | File inf; 43 | if(args.length == 2) { 44 | inf = openFileComplex(args[1], "r"); 45 | } else { 46 | inf = openFileComplex(infile, "r"); 47 | } 48 | pack = ParseProtoSchema(infile, inf.byLine.join("\n").idup); 49 | } 50 | openFileComplex(outfile, "w").write(fmt.format(pack)); 51 | } 52 | -------------------------------------------------------------------------------- /dub.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dproto", 3 | "description": "D Protocol Buffer library", 4 | "homepage": "http://github.com/msoucy/dproto", 5 | "copyright": "Copyright © 2013, Matt Soucy", 6 | "license": "BSL-1.0", 7 | "authors": [ 8 | "Matt Soucy" 9 | ], 10 | "targetType": "library", 11 | "importPaths": [ 12 | "import" 13 | ], 14 | "sourcePaths": [ 15 | "import" 16 | ], 17 | "subPackages": [ 18 | "dprotoc" 19 | ], 20 | "dependencies": { 21 | "painlessjson": {"version": "~>1.3.1", "optional": true}, 22 | "painlesstraits": {"version": "~>0.3.0"} 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /examples/proto/person.proto: -------------------------------------------------------------------------------- 1 | message Person { 2 | required string name = 1; 3 | required int32 id = 2; 4 | optional string email = 3; 5 | 6 | enum PhoneType { 7 | MOBILE = 0; 8 | HOME = 0; 9 | WORK = 2; 10 | } 11 | 12 | message PhoneNumber { 13 | required string number = 1; 14 | optional PhoneType type = 2 [default = MOBILE]; 15 | } 16 | 17 | repeated PhoneNumber phone = 4; 18 | } 19 | -------------------------------------------------------------------------------- /examples/simple.d: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env dub 2 | /+ dub.sdl: 3 | name "dproto_simple" 4 | description "A simple dproto example" 5 | dependency "dproto" path=".." 6 | author "Bjarne Leif Bruhn" 7 | author "Matt Soucy" 8 | +/ 9 | 10 | import std.stdio; 11 | import dproto.dproto; 12 | 13 | mixin ProtocolBufferFromString!" 14 | message Person { 15 | required string name = 1; 16 | required int32 id = 2; 17 | optional string email = 3; 18 | 19 | enum PhoneType { 20 | MOBILE = 0; 21 | HOME = 1; 22 | WORK = 2; 23 | } 24 | 25 | message PhoneNumber { 26 | required string number = 1; 27 | optional PhoneType type = 2 [default = HOME]; 28 | } 29 | 30 | repeated PhoneNumber phone = 4; 31 | } 32 | "; 33 | 34 | 35 | int main() 36 | { 37 | Person person; 38 | person.name = "John Doe"; 39 | person.id = 1234; 40 | person.email = "jdoe@example.com"; 41 | 42 | ubyte[] serializedObject = person.serialize(); 43 | 44 | Person person2 = Person(serializedObject); 45 | writeln("Name: ", person2.name); 46 | writeln("E-mail: ", person2.email); 47 | return 0; 48 | } 49 | -------------------------------------------------------------------------------- /examples/simple_file.d: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env dub 2 | /+ dub.sdl: 3 | name "dproto_simple" 4 | description "A simple dproto example with proto file support" 5 | dependency "dproto" path=".." 6 | author "Bjarne Leif Bruhn" 7 | author "Matt Soucy" 8 | stringImportPaths "proto" 9 | +/ 10 | 11 | /******************************************************************************* 12 | * An simple example for dproto 13 | * 14 | * Authors: Bjarne Leif Bruhn 15 | * Date: Nov 24, 2013 16 | * Version: 1.0.0 17 | */ 18 | 19 | import std.stdio; 20 | import dproto.dproto; 21 | 22 | mixin ProtocolBuffer!"person.proto"; 23 | 24 | 25 | int main() 26 | { 27 | Person person; 28 | person.name = "John Doe"; 29 | person.id = 1234; 30 | person.email = "jdoe@example.com"; 31 | 32 | ubyte[] serializedObject = person.serialize(); 33 | 34 | Person person2 = Person(serializedObject); 35 | writeln("Name: ", person2.name); 36 | writeln("E-mail: ", person2.email); 37 | return 0; 38 | } 39 | -------------------------------------------------------------------------------- /import/dproto/attributes.d: -------------------------------------------------------------------------------- 1 | /******************************************************************************* 2 | * User-Defined Attributes used to tag fields as dproto-serializable 3 | * 4 | * Authors: Matthew Soucy, dproto@msoucy.me 5 | */ 6 | module dproto.attributes; 7 | 8 | import dproto.serialize; 9 | import painlesstraits : getAnnotation, hasValueAnnotation; 10 | import dproto.compat; 11 | 12 | import std.traits : isArray; 13 | import std.typecons : Nullable; 14 | 15 | alias Identity(alias A) = A; 16 | 17 | struct ProtoField 18 | { 19 | string wireType; 20 | ubyte fieldNumber; 21 | @disable this(); 22 | this(string w, ubyte f) { 23 | wireType = w; 24 | fieldNumber = f; 25 | } 26 | @nogc auto header() { 27 | return (wireType.msgType | (fieldNumber << 3)); 28 | } 29 | } 30 | 31 | struct Required {} 32 | struct Packed {} 33 | 34 | template TagId(alias T) 35 | if(hasValueAnnotation!(T, ProtoField)) 36 | { 37 | enum TagId = getAnnotation!(T, ProtoField).fieldNumber; 38 | } 39 | 40 | template ProtoAccessors() 41 | { 42 | 43 | static auto fromProto(R)(auto ref R data) 44 | if(isProtoInputRange!R) 45 | { 46 | auto ret = typeof(this)(); 47 | ret.deserialize(data); 48 | return ret; 49 | } 50 | 51 | public this(R)(auto ref R __data) 52 | if(isProtoInputRange!R) 53 | { 54 | deserialize(__data); 55 | } 56 | 57 | ubyte[] serialize() const 58 | { 59 | import std.array : appender; 60 | auto __a = appender!(ubyte[]); 61 | serializeTo(__a); 62 | return __a.data; 63 | } 64 | 65 | void serializeTo(R)(ref R __r) const 66 | if(isProtoOutputRange!R) 67 | { 68 | import dproto.attributes; 69 | foreach(__member; ProtoFields!this) { 70 | alias __field = Identity!(__traits(getMember, this, __member)); 71 | serializeField!__field(__r); 72 | } 73 | } 74 | 75 | void deserialize(R)(auto ref R __r) 76 | if(isProtoInputRange!R) 77 | { 78 | import dproto.attributes; 79 | import painlesstraits : getAnnotation; 80 | 81 | while(!__r.empty()) { 82 | auto __msgdata = __r.readVarint(); 83 | bool __matched = false; 84 | foreach(__member; ProtoFields!this) { 85 | alias __field = Identity!(__traits(getMember, this, __member)); 86 | alias __fieldData = getAnnotation!(__field, ProtoField); 87 | if(__msgdata.msgNum == __fieldData.fieldNumber) { 88 | __r.putProtoVal!(__field)(__msgdata); 89 | __matched = true; 90 | } 91 | } 92 | if(!__matched) { 93 | defaultDecode(__msgdata, __r); 94 | } 95 | } 96 | } 97 | 98 | void mergeFrom(this This)(auto ref This rhs) 99 | { 100 | import dproto.attributes; 101 | 102 | foreach(__member; ProtoFields!this) { 103 | alias __lmember = Identity!(__traits(getMember, this, __member)); 104 | alias T = typeof(__lmember); 105 | 106 | static if(is(T : const string) || is(T : const(ubyte)[])) { 107 | __lmember = __traits(getMember, rhs, __member); 108 | } else static if(is(T : const(U)[], U)) { 109 | __lmember ~= __traits(getMember, rhs, __member); 110 | } else { 111 | __lmember = __traits(getMember, rhs, __member); 112 | } 113 | } 114 | } 115 | 116 | version(Have_painlessjson) { 117 | auto toJson() const { 118 | import painlessjson; 119 | import std.conv : to; 120 | return painlessjson.toJSON(this).to!string; 121 | } 122 | } 123 | } 124 | 125 | template ProtoFields(alias self) 126 | { 127 | import std.typetuple : Filter, TypeTuple, Erase; 128 | 129 | alias Field(alias F) = Identity!(__traits(getMember, self, F)); 130 | alias HasProtoField(alias F) = hasValueAnnotation!(Field!F, ProtoField); 131 | alias AllMembers = TypeTuple!(__traits(allMembers, typeof(self))); 132 | alias ProtoFields = Filter!(HasProtoField, Erase!("dproto", AllMembers)); 133 | } 134 | 135 | template protoDefault(T) { 136 | import std.traits : isFloatingPoint; 137 | static if(isFloatingPoint!T) { 138 | enum protoDefault = 0.0; 139 | } else static if(is(T : const string)) { 140 | enum protoDefault = ""; 141 | } else static if(is(T : const ubyte[])) { 142 | enum protoDefault = []; 143 | } else { 144 | enum protoDefault = T.init; 145 | } 146 | } 147 | 148 | void serializeField(alias field, R)(ref R r) const 149 | if (isProtoOutputRange!R) 150 | { 151 | alias fieldType = typeof(field); 152 | enum fieldData = getAnnotation!(field, ProtoField); 153 | // Serialize if required or if the value isn't the (proto) default 154 | enum isNullable = is(fieldType == Nullable!U, U); 155 | bool needsToSerialize; 156 | static if (isNullable) { 157 | needsToSerialize = ! field.isNull; 158 | } else { 159 | needsToSerialize = hasValueAnnotation!(field, Required) 160 | || (field != protoDefault!fieldType); 161 | } 162 | 163 | // If we still don't need to serialize, we're done here 164 | if (!needsToSerialize) 165 | { 166 | return; 167 | } 168 | static if(isNullable) { 169 | const rawField = field.get; 170 | } else { 171 | const rawField = field; 172 | } 173 | 174 | enum isPacked = hasValueAnnotation!(field, Packed); 175 | enum isPackType = is(fieldType == enum) || fieldData.wireType.isBuiltinType; 176 | static if (isPacked && isArray!fieldType && isPackType) 177 | alias serializer = serializePackedProto; 178 | else 179 | alias serializer = serializeProto; 180 | serializer!fieldData(rawField, r); 181 | } 182 | 183 | void putProtoVal(alias __field, R)(auto ref R r, ulong __msgdata) 184 | if (isProtoInputRange!R) 185 | { 186 | import std.range : ElementType, takeExactly, popFrontExactly; 187 | import std.array : empty; // Needed if R is an array 188 | import std.traits : isSomeString, isDynamicArray; 189 | 190 | alias T = typeof(__field); 191 | enum wireType = getAnnotation!(__field, ProtoField).wireType; 192 | enum isBinString(T) = Identity!(is(T : const(ubyte)[])); 193 | static if (isDynamicArray!T && !(isSomeString!T || isBinString!T)) 194 | { 195 | ElementType!T u; 196 | if (wireType.msgType != PACKED_MSG_TYPE && __msgdata.wireType == PACKED_MSG_TYPE) 197 | { 198 | size_t nelems = cast(size_t)r.readVarint(); 199 | auto packeddata = takeExactly(r, nelems); 200 | while(!packeddata.empty) 201 | { 202 | u.putSingleProtoVal!wireType(packeddata); 203 | __field ~= u; 204 | } 205 | r.popFrontExactly(nelems); 206 | } 207 | else 208 | { 209 | u.putSingleProtoVal!wireType(r); 210 | __field ~= u; 211 | } 212 | } 213 | else 214 | { 215 | __field.putSingleProtoVal!wireType(r); 216 | } 217 | } 218 | 219 | void putSingleProtoVal(string wireType, T, R)(ref T t, auto ref R r) 220 | if(isProtoInputRange!R) 221 | { 222 | import std.conv : to; 223 | static if(is(T : Nullable!U, U)) { 224 | U t_tmp; 225 | t_tmp.putSingleProtoVal!wireType(r); 226 | t = t_tmp; 227 | } else static if(wireType.isBuiltinType) { 228 | t = r.readProto!wireType().to!T(); 229 | } else static if(is(T == enum)) { 230 | t = r.readProto!ENUM_SERIALIZATION().to!T(); 231 | } else { 232 | auto myData = r.readProto!"bytes"(); 233 | return t.deserialize(myData); 234 | } 235 | } 236 | 237 | void serializeProto(alias fieldData, T, R)(const T data, ref R r) 238 | if(isProtoOutputRange!R) 239 | { 240 | static import dproto.serialize; 241 | static if(is(T : const string)) { 242 | r.toVarint(fieldData.header); 243 | r.writeProto!"string"(data); 244 | } else static if(is(T : const(ubyte)[])) { 245 | r.toVarint(fieldData.header); 246 | r.writeProto!"bytes"(data); 247 | } else static if(is(T : const(T)[], T)) { 248 | foreach(val; data) { 249 | serializeProto!fieldData(val, r); 250 | } 251 | } else static if(fieldData.wireType.isBuiltinType) { 252 | r.toVarint(fieldData.header); 253 | enum wt = fieldData.wireType; 254 | r.writeProto!(wt)(data); 255 | } else static if(is(T == enum)) { 256 | r.toVarint(ENUM_SERIALIZATION.msgType | (fieldData.fieldNumber << 3)); 257 | r.writeProto!ENUM_SERIALIZATION(data); 258 | } else static if(__traits(compiles, data.serialize())) { 259 | r.toVarint(fieldData.header); 260 | dproto.serialize.CntRange cnt; 261 | data.serializeTo(cnt); 262 | r.toVarint(cnt.cnt); 263 | data.serializeTo(r); 264 | } else { 265 | static assert(0, "Unknown serialization"); 266 | } 267 | } 268 | 269 | void serializePackedProto(alias fieldData, T, R)(const T data, ref R r) 270 | if(isProtoOutputRange!R) 271 | { 272 | static assert(fieldData.wireType.isBuiltinType, 273 | "Cannot have packed repeated message"); 274 | if(data.length) { 275 | dproto.serialize.CntRange cnt; 276 | static if(is(T == enum)) { 277 | enum wt = ENUM_SERIALIZATION.msgType; 278 | } else { 279 | enum wt = fieldData.wireType; 280 | } 281 | foreach (ref e; data) 282 | cnt.writeProto!wt(e); 283 | toVarint(r, PACKED_MSG_TYPE | (fieldData.fieldNumber << 3)); 284 | toVarint(r, cnt.cnt); 285 | foreach (ref e; data) 286 | r.writeProto!wt(e); 287 | } 288 | } 289 | -------------------------------------------------------------------------------- /import/dproto/compat.d: -------------------------------------------------------------------------------- 1 | /******************************************************************************* 2 | * Compatibility layer for different D/Phobos versions 3 | * 4 | * Authors: Matthew Soucy, dproto@msoucy.me 5 | */ 6 | module dproto.compat; 7 | 8 | // nogc compat shim using UDAs (@nogc must appear as function prefix) 9 | static if (__VERSION__ < 2066) enum nogc; 10 | 11 | enum DPROTO_PROTOBUF_VERSION = 2.2; 12 | -------------------------------------------------------------------------------- /import/dproto/dproto.d: -------------------------------------------------------------------------------- 1 | /******************************************************************************* 2 | * Main library import for dproto 3 | * 4 | * Provides accessors for D string and D structs from proto files/data 5 | * 6 | * Authors: Matthew Soucy, dproto@msoucy.me 7 | */ 8 | module dproto.dproto; 9 | 10 | import std.exception : enforce; 11 | import std.array; 12 | import std.range; 13 | 14 | 15 | /******************************************************************************* 16 | * Create D structures from proto file 17 | * 18 | * Creates all required structs given a valid proto file 19 | * 20 | * Assumes that the file can be found in the string imports 21 | */ 22 | template ProtocolBuffer(string s) 23 | { 24 | import dproto.imports; 25 | mixin(ParseProtoSchema(s,import(s)).toD()); 26 | } 27 | 28 | /******************************************************************************* 29 | * Create D structure strings from proto data 30 | * 31 | * Creates all required structs given a valid proto definition as a string 32 | */ 33 | template ProtocolBufferFromString(string s) 34 | { 35 | import dproto.imports; 36 | mixin(ParseProtoSchema("",s).toD()); 37 | } 38 | 39 | template ProtocolBufferInterface(string s) { 40 | import dproto.imports; 41 | mixin("%3.1s".format(ParseProtoSchema("",s))); 42 | } 43 | 44 | template ProtocolBufferRpc(string s) { 45 | import dproto.imports; 46 | mixin("%3.2s".format(ParseProtoSchema("",s))); 47 | } 48 | 49 | template ProtocolBufferImpl(string s) { 50 | import dproto.imports; 51 | mixin("%3.3s".format(ParseProtoSchema("",s))); 52 | } 53 | 54 | template ProtocolBufferStruct(string s) { 55 | import dproto.imports; 56 | mixin("%-3.1s".format(ParseProtoSchema("",s))); 57 | } 58 | 59 | -------------------------------------------------------------------------------- /import/dproto/exception.d: -------------------------------------------------------------------------------- 1 | /******************************************************************************* 2 | * Exceptions used by the D protocol buffer system 3 | * 4 | * Authors: Matthew Soucy, dproto@msoucy.me 5 | */ 6 | module dproto.exception; 7 | import std.exception; 8 | 9 | /// Basic exception, something went wrong with creating a buffer struct 10 | class DProtoException : Exception { 11 | static if (__traits(compiles, {mixin basicExceptionCtors;})) { 12 | /// 13 | mixin basicExceptionCtors; 14 | } else { 15 | this(string msg, string file = __FILE__, size_t line = __LINE__, 16 | Throwable next = null) @safe pure nothrow { 17 | super(msg, file, line, next); 18 | } 19 | 20 | this(string msg, Throwable next, string file = __FILE__, 21 | size_t line = __LINE__) @safe pure nothrow { 22 | super(msg, file, line, next); 23 | } 24 | } 25 | } 26 | 27 | /// Proto file attempted to use a reserved word 28 | class DProtoReservedWordException : DProtoException { 29 | static if (__traits(compiles, {mixin basicExceptionCtors;})) { 30 | /// 31 | mixin basicExceptionCtors; 32 | } else { 33 | this(string msg, string file = __FILE__, size_t line = __LINE__, 34 | Throwable next = null) @safe pure nothrow { 35 | super(msg, file, line, next); 36 | } 37 | 38 | this(string msg, Throwable next, string file = __FILE__, 39 | size_t line = __LINE__) @safe pure nothrow { 40 | super(msg, file, line, next); 41 | } 42 | } 43 | } 44 | 45 | /// Proto file used invalid syntax 46 | class DProtoSyntaxException : DProtoException { 47 | static if (__traits(compiles, {mixin basicExceptionCtors;})) { 48 | /// 49 | mixin basicExceptionCtors; 50 | } else { 51 | this(string msg, string file = __FILE__, size_t line = __LINE__, 52 | Throwable next = null) @safe pure nothrow { 53 | super(msg, file, line, next); 54 | } 55 | 56 | this(string msg, Throwable next, string file = __FILE__, 57 | size_t line = __LINE__) @safe pure nothrow { 58 | super(msg, file, line, next); 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /import/dproto/imports.d: -------------------------------------------------------------------------------- 1 | module dproto.imports; 2 | 3 | public import dproto.attributes; 4 | public import dproto.exception; 5 | public import dproto.serialize; 6 | public import dproto.parse; 7 | 8 | public import std.range; 9 | public import std.string; 10 | public import std.format; 11 | 12 | -------------------------------------------------------------------------------- /import/dproto/intermediate.d: -------------------------------------------------------------------------------- 1 | /******************************************************************************* 2 | * Intermediate structures used for generating class strings 3 | * 4 | * These are only used internally, so are not being exported 5 | * 6 | * Authors: Matthew Soucy, dproto@msoucy.me 7 | */ 8 | module dproto.intermediate; 9 | 10 | import dproto.attributes; 11 | import dproto.serialize; 12 | 13 | import std.algorithm; 14 | import std.conv; 15 | import std.string; 16 | import std.format; 17 | 18 | package: 19 | 20 | struct Options { 21 | string[string] raw; 22 | alias raw this; 23 | void toString(scope void delegate(const(char)[]) sink, FormatSpec!char fmt) const 24 | { 25 | if(fmt.spec == 'p') { 26 | if(!raw.length) return; 27 | sink.formattedWrite(" [%-(%s = %s%|, %)]", raw); 28 | } else { 29 | sink.formattedWrite(`["dproto_generated": "true"%(, %s : %s%)]`, raw); 30 | } 31 | } 32 | } 33 | 34 | struct MessageType { 35 | this(string name) { 36 | this.name = name; 37 | } 38 | string name; 39 | Options options; 40 | Field[] fields; 41 | EnumType[] enumTypes; 42 | MessageType[] messageTypes; 43 | 44 | void toString(scope void delegate(const(char)[]) sink, FormatSpec!char fmt) const 45 | { 46 | if(fmt.spec == 'p') { 47 | sink.formattedWrite("message %s { ", name); 48 | foreach(opt, val; options) { 49 | sink.formattedWrite("option %s = %s; ", opt, val); 50 | } 51 | } else { 52 | sink.formattedWrite("static struct %s {\n", name); 53 | 54 | // Methods for serialization and deserialization. 55 | sink("static import dproto.attributes;\n"); 56 | sink("mixin dproto.attributes.ProtoAccessors;\n"); 57 | } 58 | foreach(et; enumTypes) et.toString(sink, fmt); 59 | foreach(mt; messageTypes) mt.toString(sink, fmt); 60 | foreach(field; fields) field.toString(sink, fmt); 61 | sink("}\n"); 62 | } 63 | 64 | string toD() @property const { return "%s".format(this); } 65 | } 66 | 67 | struct EnumType { 68 | this(string name) { 69 | this.name = name.idup; 70 | } 71 | string name; 72 | Options options; 73 | int[string] values; 74 | 75 | void toString(scope void delegate(const(char)[]) sink, FormatSpec!char fmt) const 76 | { 77 | sink.formattedWrite("enum %s {\n", name); 78 | string suffix = ", "; 79 | if(fmt.spec == 'p') { 80 | foreach(opt, val; options) { 81 | sink.formattedWrite("option %s = %s; ", opt, val); 82 | } 83 | suffix = "; "; 84 | } 85 | foreach(key, val; values) { 86 | sink.formattedWrite("%s = %s%s", key, val, suffix); 87 | } 88 | sink("}\n"); 89 | } 90 | 91 | string toD() @property const { return "%s".format(this); } 92 | } 93 | 94 | struct Option { 95 | this(string name, string value) { 96 | this.name = name.idup; 97 | this.value = value.idup; 98 | } 99 | string name; 100 | string value; 101 | } 102 | 103 | struct Extension { 104 | ulong minVal = 0; 105 | ulong maxVal = ulong.max; 106 | } 107 | 108 | struct Dependency { 109 | this(string depname, bool isPublic = false, bool isWeak = false) { 110 | this.name = depname; 111 | this.isPublic = isPublic; 112 | this.isWeak = isWeak; 113 | } 114 | string name; 115 | bool isPublic; 116 | bool isWeak; 117 | 118 | void toString(scope void delegate(const(char)[]) sink, FormatSpec!char fmt) const 119 | { 120 | if(fmt.spec == 'p') { 121 | sink("import "); 122 | if(isPublic) { 123 | sink("public "); 124 | } 125 | sink.formattedWrite(`"%s"; `, name); 126 | } else { 127 | sink.formattedWrite(`mixin ProtocolBuffer!"%s";`, name); 128 | } 129 | } 130 | 131 | string toD() @property const { return "%s".format(this); } 132 | } 133 | 134 | struct ProtoPackage { 135 | this(string fileName) { 136 | this.fileName = fileName.idup; 137 | } 138 | string fileName; 139 | string packageName; 140 | Dependency[] dependencies; 141 | EnumType[] enumTypes; 142 | MessageType[] messageTypes; 143 | Options options; 144 | Service[] rpcServices; 145 | string syntax; 146 | 147 | void toString(scope void delegate(const(char)[]) sink, FormatSpec!char fmt) const 148 | { 149 | if(fmt.spec == 'p') { 150 | if(packageName) { 151 | sink.formattedWrite("package %s; ", packageName); 152 | } 153 | if(syntax !is null && syntax != "proto2") { 154 | sink.formattedWrite(`syntax = %s; `, syntax); 155 | } 156 | } 157 | foreach(dep; dependencies) { 158 | dep.toString(sink, fmt); 159 | } 160 | foreach(e; enumTypes) e.toString(sink, fmt); 161 | foreach(m; messageTypes) m.toString(sink, fmt); 162 | foreach(r; rpcServices) r.toString(sink, fmt); 163 | if(fmt.spec == 'p') { 164 | foreach(opt, val; options) { 165 | sink.formattedWrite("option %s = %s; ", opt, val); 166 | } 167 | } 168 | } 169 | 170 | string toProto() @property const { return "%p".format(this); } 171 | string toD() @property const { return "%s".format(this); } 172 | } 173 | 174 | struct Field { 175 | enum Requirement { 176 | OPTIONAL, 177 | REPEATED, 178 | REQUIRED 179 | } 180 | this(Requirement labelEnum, string type, string name, uint tag, Options options) { 181 | this.requirement = labelEnum; 182 | this.type = type; 183 | this.name = name; 184 | this.id = tag; 185 | this.options = options; 186 | } 187 | Requirement requirement; 188 | string type; 189 | string name; 190 | uint id; 191 | Options options; 192 | 193 | bool hasDefaultValue() const{ 194 | return null != ("default" in options); 195 | } 196 | string defaultValue() const { 197 | return options["default"]; 198 | } 199 | 200 | void toString(scope void delegate(const(char)[]) sink, FormatSpec!char fmt) const 201 | { 202 | switch(fmt.spec) { 203 | case 'p': 204 | if(fmt.width == 3 && requirement != Requirement.REPEATED) { 205 | sink.formattedWrite("%s %s = %s%p; ", 206 | type, name, id, options); 207 | } 208 | else { 209 | sink.formattedWrite("%s %s %s = %s%p; ", 210 | requirement.to!string.toLower(), 211 | type, name, id, options); 212 | } 213 | break; 214 | default: 215 | if(!fmt.flDash) { 216 | getDeclaration(sink); 217 | } else { 218 | sink.formattedWrite("%s %s;\n", 219 | type, name); 220 | } 221 | break; 222 | } 223 | } 224 | 225 | void getDeclaration(scope void delegate(const(char)[]) sink) const { 226 | if(requirement == Requirement.REQUIRED) { 227 | sink("@(dproto.attributes.Required())\n"); 228 | } else if(requirement == Requirement.REPEATED) { 229 | if(options.get("packed", "false") != "false") { 230 | sink("@(dproto.attributes.Packed())\n"); 231 | } 232 | } 233 | sink("@(dproto.attributes.ProtoField"); 234 | sink.formattedWrite(`("%s", %s)`, type, id); 235 | sink(")\n"); 236 | 237 | bool wrap_with_nullable = 238 | requirement == Requirement.OPTIONAL && 239 | ! type.isBuiltinType(); 240 | 241 | if(wrap_with_nullable) { 242 | sink(`dproto.serialize.PossiblyNullable!(`); 243 | } 244 | string typestr = type; 245 | if(type.isBuiltinType) { 246 | typestr = format(`BuffType!"%s"`, type); 247 | } 248 | 249 | sink(typestr); 250 | 251 | if(wrap_with_nullable) { 252 | sink(`)`); 253 | } 254 | if(requirement == Requirement.REPEATED) { 255 | sink("[]"); 256 | } 257 | sink.formattedWrite(" %s", name); 258 | if (requirement != Requirement.REPEATED) { 259 | if (hasDefaultValue) { 260 | sink.formattedWrite(`= SpecifiedDefaultValue!(%s, "%s")`, typestr, defaultValue); 261 | } else if(type.isBuiltinType || ! wrap_with_nullable) { 262 | sink.formattedWrite("= UnspecifiedDefaultValue!(%s)", typestr); 263 | } 264 | } 265 | sink(";\n\n"); 266 | } 267 | void getCase(scope void delegate(const(char)[]) sink) const { 268 | sink.formattedWrite("case %s:\n", id); 269 | sink.formattedWrite("%s.deserialize(__msgdata, __data);\n", name); 270 | if(requirement == Requirement.REQUIRED) { 271 | sink.formattedWrite("%s_isset = true;\n", name); 272 | } 273 | sink("break;\n"); 274 | } 275 | } 276 | 277 | 278 | struct Service { 279 | this(string name) { 280 | this.name = name.idup; 281 | } 282 | string name; 283 | Options options; 284 | Method[] rpc; 285 | 286 | struct Method { 287 | this(string name, string documentation, string requestType, string responseType) { 288 | this.name = name.idup; 289 | this.documentation = documentation.idup; 290 | this.requestType = requestType.idup; 291 | this.responseType = responseType.idup; 292 | } 293 | string name; 294 | string documentation; 295 | string requestType; 296 | string responseType; 297 | Options options; 298 | 299 | void toString(scope void delegate(const(char)[]) sink, FormatSpec!char fmt) const 300 | { 301 | switch(fmt.spec) { 302 | case 'p': 303 | sink.formattedWrite("rpc %s (%s) returns (%s)", name, requestType, responseType); 304 | if(options.length > 0) { 305 | sink(" {\n"); 306 | foreach(opt, val; options) { 307 | sink.formattedWrite("option %s = %s;\n", opt, val); 308 | } 309 | sink("}\n"); 310 | } else { 311 | sink(";\n"); 312 | } 313 | break; 314 | default: 315 | if(fmt.precision == 3) { 316 | sink.formattedWrite("%s %s (%s) { %s res; return res; }\n", responseType, name, requestType, responseType); 317 | } else if(fmt.precision == 2) { 318 | sink.formattedWrite("void %s (const %s, ref %s);\n", name, requestType, responseType); 319 | } else if(fmt.precision == 1) { 320 | sink.formattedWrite("%s %s (%s);\n", responseType, name, requestType); 321 | } 322 | break; 323 | } 324 | } 325 | } 326 | 327 | void toString(scope void delegate(const(char)[]) sink, FormatSpec!char fmt) const 328 | { 329 | switch(fmt.spec) { 330 | case 'p': 331 | sink.formattedWrite("service %s {\n", name); 332 | break; 333 | default: 334 | if(fmt.precision == 3) { 335 | sink.formattedWrite("class %s {\n", name); 336 | } else if(fmt.precision == 2 || fmt.precision == 1) { 337 | sink.formattedWrite("interface %s {\n", name); 338 | } else { 339 | return; 340 | } 341 | break; 342 | } 343 | foreach(m; rpc) m.toString(sink, fmt); 344 | sink("}\n"); 345 | } 346 | } 347 | -------------------------------------------------------------------------------- /import/dproto/package.d: -------------------------------------------------------------------------------- 1 | /******************************************************************************* 2 | * Package import for dproto 3 | * 4 | * Imports the required components for the structure 5 | * 6 | * Authors: Matthew Soucy, dproto@msoucy.me 7 | */ 8 | module dproto; 9 | public import dproto.dproto; 10 | public import dproto.unittests; 11 | -------------------------------------------------------------------------------- /import/dproto/parse.d: -------------------------------------------------------------------------------- 1 | /******************************************************************************* 2 | * Convert a .proto file into a string representing the class 3 | * 4 | * Author: Matthew Soucy, dproto@msoucy.me 5 | */ 6 | module dproto.parse; 7 | 8 | import dproto.exception; 9 | import dproto.intermediate; 10 | import dproto.serialize : isBuiltinType; 11 | 12 | import std.algorithm; 13 | import std.array; 14 | import std.ascii; 15 | import std.conv; 16 | import std.exception; 17 | import std.format; 18 | import std.stdio; 19 | import std.string; 20 | import std.traits; 21 | 22 | enum wordPattern = std.ascii.letters ~ std.ascii.digits ~ `_.\-`; 23 | enum pathPattern = wordPattern ~ `/`; 24 | 25 | /** 26 | * Basic parser for {@code .proto} schema declarations. 27 | * 28 | *

This parser throws away data that it doesn't care about. In particular, 29 | * unrecognized options, and extensions are discarded. It doesn't retain nesting 30 | * within types. 31 | */ 32 | ProtoPackage ParseProtoSchema(const string name_, string data_) 33 | { 34 | 35 | struct ProtoSchemaParser { 36 | 37 | /** The path to the {@code .proto} file. */ 38 | string fileName; 39 | 40 | /** The entire document. */ 41 | const char[] data; 42 | 43 | /** Our cursor within the document. {@code data[pos]} is the next character to be read. */ 44 | int pos; 45 | 46 | /** The number of newline characters encountered thus far. */ 47 | int line; 48 | 49 | /** The index of the most recent newline character. */ 50 | int lineStart; 51 | 52 | /** Are we parsing proto 3 syntax? */ 53 | bool isProto3; 54 | 55 | ProtoPackage readProtoPackage() { 56 | auto ret = ProtoPackage(fileName); 57 | while (true) { 58 | readDocumentation(); 59 | if (pos == data.length) { 60 | return ret; 61 | } 62 | readDeclaration(ret); 63 | } 64 | } 65 | 66 | this(string _fileName, string _data) 67 | { 68 | fileName = _fileName; 69 | data = _data; 70 | } 71 | 72 | private: 73 | 74 | void readDeclaration(Context, string ContextName = Context.stringof)(ref Context context) { 75 | // Skip unnecessary semicolons, occasionally used after a nested message declaration. 76 | if (peekChar() == ';') { 77 | pos++; 78 | return; 79 | } 80 | 81 | string label = readWord(); 82 | 83 | switch(label) { 84 | case "syntax": { 85 | static if(is(Context==ProtoPackage)) { 86 | unexpected(context.syntax == null, "Too many syntax statements"); 87 | unexpected(readChar() == '=', "Expected '=' after 'syntax'"); 88 | unexpected(peekChar() == '"', `Expected opening quote '"' after 'syntax ='`); 89 | context.syntax = readQuotedString(); 90 | unexpected(context.syntax == `"proto2"` || context.syntax == `"proto3"`, 91 | "Unexpected syntax version: `" ~ context.syntax ~ "`"); 92 | isProto3 = context.syntax == `"proto3"`; 93 | unexpected(readChar() == ';', "Expected ';' after syntax declaration"); 94 | return; 95 | } else { 96 | throw new DProtoSyntaxException("syntax in " ~ ContextName); 97 | } 98 | } 99 | case "package": { 100 | static if(is(Context==ProtoPackage)) { 101 | unexpected(context.packageName == null, "too many package names"); 102 | context.packageName = readSymbolName(context); 103 | unexpected(readChar() == ';', "Expected ';'"); 104 | return; 105 | } else { 106 | throw new DProtoSyntaxException("package in " ~ ContextName); 107 | } 108 | } 109 | case "import": { 110 | static if(is(Context==ProtoPackage)) { 111 | bool isPublicImport = false; 112 | bool isWeakImport = false; 113 | if(peekChar() == 'p') { 114 | unexpected(readWord() == "public", "Expected 'public'"); 115 | isPublicImport = true; 116 | } else if(peekChar() == 'w') { 117 | unexpected(readWord() == "weak", "Expected 'weak'"); 118 | isWeakImport = true; 119 | } 120 | if(peekChar() == '"') { 121 | context.dependencies ~= Dependency(readQuotedPath (), isPublicImport, isWeakImport); 122 | } 123 | unexpected(readChar() == ';', "Expected ';'"); 124 | return; 125 | } else { 126 | throw new DProtoSyntaxException("import in " ~ ContextName); 127 | } 128 | } 129 | case "option": { 130 | Option result = readOption('='); 131 | unexpected(readChar() == ';', "Expected ';'"); 132 | context.options[result.name] = result.value; 133 | return; 134 | } 135 | case "message": { 136 | static if(hasMember!(Context, "messageTypes")) { 137 | context.messageTypes ~= readMessage(context); 138 | return; 139 | } else { 140 | throw new DProtoSyntaxException("message in " ~ ContextName); 141 | } 142 | } 143 | case "enum": { 144 | static if(hasMember!(Context, "enumTypes")) { 145 | context.enumTypes ~= readEnumType(context); 146 | return; 147 | } else { 148 | throw new DProtoSyntaxException("enum in " ~ ContextName); 149 | } 150 | } 151 | case "extend": { 152 | readExtend(); 153 | return; 154 | } 155 | case "service": { 156 | static if(hasMember!(Context, "rpcServices")) { 157 | context.rpcServices ~= readService(context); 158 | return; 159 | } else { 160 | throw new DProtoSyntaxException("service in " ~ ContextName); 161 | } 162 | } 163 | case "rpc": { 164 | static if( hasMember!(Context, "rpc")) { 165 | context.rpc ~= readRpc(context); 166 | return; 167 | } else { 168 | throw new DProtoSyntaxException("rpc in " ~ ContextName); 169 | } 170 | } 171 | case "required": 172 | case "optional": 173 | if( isProto3 ) { 174 | throw new DProtoSyntaxException("Field label '" ~ label ~ "' not allowed"); 175 | } 176 | goto case; 177 | case "repeated": { 178 | static if( hasMember!(Context, "fields") ) { 179 | string type = readSymbolName(context); 180 | auto newfield = readField(label, type, context); 181 | unexpected(context.fields.all!(a => a.id != newfield.id)(), 182 | "Repeated field ID"); 183 | context.fields ~= newfield; 184 | return; 185 | } else { 186 | throw new DProtoSyntaxException("Fields must be nested"); 187 | } 188 | } 189 | case "map": 190 | case "oneof": { 191 | throw new DProtoSyntaxException("'" ~ label ~ "' not yet implemented"); 192 | } 193 | case "extensions": { 194 | static if(!is(Context==ProtoPackage)) { 195 | readExtensions(context); 196 | return; 197 | } else { 198 | throw new DProtoSyntaxException("Extensions must be nested"); 199 | } 200 | } 201 | default: { 202 | static if (is(Context == EnumType)) 203 | { 204 | unexpected(readChar() == '=', "Expected '='"); 205 | int tag = readInt(); 206 | if (context.options.get("allow_alias", "true") == "false" 207 | && context.values.values.canFind(tag)) 208 | { 209 | throw new DProtoSyntaxException("Enum values must not be duplicated"); 210 | } 211 | unexpected(readChar() == ';', "Expected ';'"); 212 | context.values[label] = tag; 213 | return; 214 | } 215 | else static if (hasMember!(Context, "fields")) 216 | { 217 | string type = reservedName(context, label); 218 | auto newfield = readField("optional", type, context); 219 | unexpected(context.fields.all!(a => a.id != newfield.id)(), 220 | "Repeated field ID"); 221 | context.fields ~= newfield; 222 | return; 223 | } 224 | else 225 | { 226 | throw new DProtoSyntaxException("unexpected label: `" ~ label ~ '`'); 227 | } 228 | } 229 | } 230 | } 231 | 232 | /** Reads a message declaration. */ 233 | MessageType readMessage(Context)(Context context) { 234 | auto ret = MessageType(readSymbolName(context)); 235 | ret.options = context.options; 236 | unexpected(readChar() == '{', "Expected '{'"); 237 | while (true) { 238 | readDocumentation(); 239 | if (peekChar() == '}') { 240 | pos++; 241 | break; 242 | } 243 | readDeclaration(ret); 244 | } 245 | return ret; 246 | } 247 | 248 | /** Reads an extend declaration (just ignores the content). 249 | @todo */ 250 | void readExtend() { 251 | readName(); // Ignore this for now 252 | unexpected(readChar() == '{', "Expected '{'"); 253 | while (true) { 254 | readDocumentation(); 255 | if (peekChar() == '}') { 256 | pos++; 257 | break; 258 | } 259 | //readDeclaration(); 260 | } 261 | return; 262 | } 263 | 264 | /** Reads a service declaration and returns it. */ 265 | Service readService(Context)(Context context) { 266 | string name = readSymbolName(context); 267 | auto ret = Service(name); 268 | 269 | Service.Method[] methods = []; 270 | unexpected(readChar() == '{', "Expected '{'"); 271 | while (true) { 272 | readDocumentation(); 273 | if (peekChar() == '}') { 274 | pos++; 275 | break; 276 | } 277 | readDeclaration(ret); 278 | } 279 | return ret; 280 | } 281 | 282 | 283 | /** Reads an rpc method and returns it. */ 284 | Service.Method readRpc(Context)(Context context) { 285 | string documentation = ""; 286 | string name = readSymbolName(context); 287 | 288 | unexpected(readChar() == '(', "Expected '('"); 289 | string requestType = readSymbolName(context); 290 | unexpected(readChar() == ')', "Expected ')'"); 291 | 292 | unexpected(readWord() == "returns", "Expected 'returns'"); 293 | 294 | unexpected(readChar() == '(', "Expected '('"); 295 | string responseType = readSymbolName(context); 296 | // @todo check for option prefixes, responseType is the last in the white spaced list 297 | unexpected(readChar() == ')', "Expected ')'"); 298 | 299 | auto ret = Service.Method(name, documentation, requestType, responseType); 300 | 301 | /* process service options and documentation */ 302 | if (peekChar() == '{') { 303 | pos++; 304 | while (true) { 305 | readDocumentation(); 306 | if (peekChar() == '}') { 307 | pos++; 308 | break; 309 | } 310 | readDeclaration(ret); 311 | } 312 | } 313 | else if (readChar() != ';') { 314 | throw new DProtoSyntaxException("Expected ';'"); 315 | } 316 | return ret; 317 | } 318 | 319 | /** Reads an enumerated type declaration and returns it. */ 320 | EnumType readEnumType(Context)(Context context) { 321 | auto ret = EnumType(readSymbolName(context)); 322 | unexpected(readChar() == '{', "Expected '{'"); 323 | while (true) { 324 | readDocumentation(); 325 | if (peekChar() == '}') { 326 | pos++; 327 | break; 328 | } 329 | readDeclaration(ret); 330 | } 331 | return ret; 332 | } 333 | 334 | /** Reads a field declaration and returns it. */ 335 | Field readField(Context)(string label, string type, Context context) { 336 | Field.Requirement labelEnum = label.toUpper().to!(Field.Requirement)(); 337 | string name = readSymbolName(context); 338 | unexpected(readChar() == '=', "Expected '='"); 339 | int tag = readInt(); 340 | enforce((0 < tag && tag < 19000) || (19999 < tag && tag < 2^^29), 341 | new DProtoSyntaxException( 342 | "Invalid tag number: "~tag.to!string())); 343 | char c = peekChar(); 344 | Options options; 345 | if (c == '[') { 346 | options = readMap('[', ']', '='); 347 | c = peekChar(); 348 | } 349 | if (c == ';') { 350 | pos++; 351 | if (labelEnum != Field.Requirement.REPEATED && options.get("packed", "false") != "false") { 352 | throw new DProtoSyntaxException("[packed = true] can only be specified for repeated primitive fields"); 353 | } 354 | return Field(labelEnum, type, name, tag, options); 355 | } 356 | throw new DProtoSyntaxException("Expected ';'"); 357 | } 358 | 359 | /** Reads extensions like "extensions 101;" or "extensions 101 to max;". 360 | @todo */ 361 | Extension readExtensions(Context)(Context context) { 362 | Extension ret; 363 | int minVal = readInt(); // Range start. 364 | if (peekChar() != ';') { 365 | unexpected(readWord() == "to", "Expected 'to'"); 366 | string maxVal = readWord(); // Range end. 367 | if(maxVal != "max") { 368 | if(maxVal[0..2] == "0x") { 369 | ret.maxVal = maxVal[2..$].to!uint(16); 370 | } else { 371 | ret.maxVal = maxVal.to!uint(); 372 | } 373 | } 374 | } else { 375 | ret.minVal = minVal; 376 | ret.maxVal = minVal; 377 | } 378 | unexpected(readChar() == ';', "Expected ';'"); 379 | return ret; 380 | } 381 | 382 | /** Reads a option containing a name, an '=' or ':', and a value. */ 383 | Option readOption(char keyValueSeparator) { 384 | string name = readName(); // Option name. 385 | unexpected(readChar() == keyValueSeparator, "Expected '" ~ keyValueSeparator ~ "' in option"); 386 | string value = (peekChar() == '{') ? readMap('{', '}', ':').to!string() : readString(); 387 | return Option(name, value); 388 | } 389 | 390 | /** 391 | * Returns a map of string keys and values. This is similar to a JSON object, 392 | * with '{' and '}' surrounding the map, ':' separating keys from values, and 393 | * ',' separating entries. 394 | */ 395 | Options readMap(char openBrace, char closeBrace, char keyValueSeparator) { 396 | unexpected(readChar() == openBrace, openBrace ~ " to begin map"); 397 | Options result; 398 | while (peekChar() != closeBrace) { 399 | 400 | Option option = readOption(keyValueSeparator); 401 | result[option.name] = option.value; 402 | 403 | char c = peekChar(); 404 | if (c == ',') { 405 | pos++; 406 | } else if (c != closeBrace) { 407 | throw new DProtoSyntaxException("Expected ',' or '" ~ closeBrace ~ "'"); 408 | } 409 | } 410 | 411 | // If we see the close brace, finish immediately. This handles {}/[] and ,}/,] cases. 412 | pos++; 413 | return result; 414 | } 415 | 416 | private: 417 | 418 | /** Reads a non-whitespace character and returns it. */ 419 | char readChar() { 420 | char result = peekChar(); 421 | pos++; 422 | return result; 423 | } 424 | 425 | /** 426 | * Peeks a non-whitespace character and returns it. The only difference 427 | * between this and {@code readChar} is that this doesn't consume the char. 428 | */ 429 | char peekChar() { 430 | skipWhitespace(true); 431 | unexpected(pos != data.length, "unexpected end of file"); 432 | return data[pos]; 433 | } 434 | 435 | /** Reads a quoted or unquoted string and returns it. */ 436 | string readString() { 437 | skipWhitespace(true); 438 | return peekChar() == '"' ? readQuotedString() : readWord(); 439 | } 440 | 441 | string readQuotedString() { 442 | skipWhitespace(true); 443 | auto c = readChar(); 444 | enforce(c == '"', new DProtoSyntaxException("Expected \" but got " ~ c)); 445 | string result; 446 | while (pos < data.length) { 447 | c = data[pos++]; 448 | if (c == '"') return '"'~result~'"'; 449 | 450 | if (c == '\\') { 451 | unexpected(pos != data.length, "unexpected end of file"); 452 | c = data[pos++]; 453 | } 454 | 455 | result ~= c; 456 | if (c == '\n') newline(); 457 | } 458 | throw new DProtoSyntaxException("unterminated string"); 459 | } 460 | 461 | string readQuotedPath() { 462 | skipWhitespace(true); 463 | unexpected(readChar() == '"', "imports should be quoted"); 464 | auto ret = readWord(pathPattern); 465 | unexpected(readChar() == '"', "imports should be quoted"); 466 | return ret; 467 | } 468 | 469 | /** Reads a (paren-wrapped), [square-wrapped] or naked symbol name. */ 470 | string readName() { 471 | string optionName; 472 | char c = peekChar(); 473 | if (c == '(') { 474 | pos++; 475 | optionName = readWord(); 476 | unexpected(readChar() == ')', "Expected ')'"); 477 | } else if (c == '[') { 478 | pos++; 479 | optionName = readWord(); 480 | unexpected(readChar() == ']', "Expected ']'"); 481 | } else { 482 | optionName = readWord(); 483 | } 484 | return optionName; 485 | } 486 | 487 | /** Reads a symbol name */ 488 | string readSymbolName(Context)(Context context) { 489 | string name = readWord(); 490 | return reservedName(context, name); 491 | } 492 | 493 | /** Format a reserved D name */ 494 | string reservedName(Context)(Context context, string name) { 495 | if(isDKeyword(name)) 496 | { 497 | // Wrapped in quotes to properly evaluate string 498 | string reservedFmtRaw = context.options.get("dproto_reserved_fmt", `"%s_"`); 499 | string reservedFmt; 500 | formattedRead(reservedFmtRaw, `"%s"`, &reservedFmt); 501 | if(reservedFmt != "%s") 502 | { 503 | name = reservedFmt.format(name); 504 | } 505 | else 506 | { 507 | throw new DProtoReservedWordException("Reserved word: "~name); 508 | } 509 | } 510 | return name; 511 | } 512 | 513 | /** Reads a non-empty word and returns it. */ 514 | string readWord(string pattern = wordPattern) { 515 | skipWhitespace(true); 516 | int start = pos; 517 | while (pos < data.length) { 518 | char c = data[pos]; 519 | if(pattern.canFind(c)) { 520 | pos++; 521 | } else { 522 | break; 523 | } 524 | } 525 | unexpected(start != pos, "Expected a word"); 526 | return data[start .. pos].idup; 527 | } 528 | 529 | /** Reads an integer and returns it. */ 530 | int readInt() { 531 | string tag = readWord(); 532 | try { 533 | int radix = 10; 534 | if (tag.startsWith("0x")) { 535 | tag = tag["0x".length .. $]; 536 | radix = 16; 537 | } 538 | else if (tag.startsWith("0")) { 539 | radix = 8; 540 | } 541 | return tag.to!int(radix); 542 | } catch (Exception e) { 543 | throw new DProtoSyntaxException( 544 | "Expected an integer but was `" ~ tag ~ "`", 545 | e.msg); 546 | } 547 | } 548 | 549 | /** 550 | * Like {@link #skipWhitespace}, but this returns a string containing all 551 | * comment text. By convention, comments before a declaration document that 552 | * declaration. 553 | */ 554 | string readDocumentation() { 555 | string result = null; 556 | while (true) { 557 | skipWhitespace(false); 558 | if (pos == data.length || data[pos] != '/') { 559 | return result != null ? cleanUpDocumentation(result) : ""; 560 | } 561 | string comment = readComment(); 562 | result = (result == null) ? comment : (result ~ "\n" ~ comment); 563 | } 564 | } 565 | 566 | /** Reads a comment and returns its body. */ 567 | string readComment() { 568 | enforce(!(pos == data.length || data[pos] != '/'), new DProtoSyntaxException("")); 569 | pos++; 570 | int commentType = pos < data.length ? data[pos++] : -1; 571 | if (commentType == '*') { 572 | int start = pos; 573 | while (pos + 1 < data.length) { 574 | if (data[pos] == '*' && data[pos + 1] == '/') { 575 | pos += 2; 576 | return data[start .. pos - 2].idup; 577 | } else { 578 | char c = data[pos++]; 579 | if (c == '\n') newline(); 580 | } 581 | } 582 | throw new DProtoSyntaxException("unterminated comment"); 583 | } else if (commentType == '/') { 584 | int start = pos; 585 | while (pos < data.length) { 586 | char c = data[pos++]; 587 | if (c == '\n') { 588 | newline(); 589 | break; 590 | } 591 | } 592 | return data[start .. pos - 1].idup; 593 | } else { 594 | throw new DProtoSyntaxException("unexpected '/'"); 595 | } 596 | } 597 | 598 | /** 599 | * Returns a string like {@code comment}, but without leading whitespace or 600 | * asterisks. 601 | */ 602 | string cleanUpDocumentation(string comment) { 603 | string result; 604 | bool beginningOfLine = true; 605 | for (int i = 0; i < comment.length; i++) { 606 | char c = comment[i]; 607 | if (!beginningOfLine || ! " \t*".canFind(c)) { 608 | result ~= c; 609 | beginningOfLine = false; 610 | } 611 | if (c == '\n') { 612 | beginningOfLine = true; 613 | } 614 | } 615 | return result.strip(); 616 | } 617 | 618 | /** 619 | * Skips whitespace characters and optionally comments. When this returns, 620 | * either {@code pos == data.length} or a non-whitespace character. 621 | */ 622 | void skipWhitespace(bool skipComments) { 623 | while (pos < data.length) { 624 | char c = data[pos]; 625 | if (" \t\r\n".canFind(c)) { 626 | pos++; 627 | if (c == '\n') newline(); 628 | } else if (skipComments && c == '/') { 629 | readComment(); 630 | } else { 631 | break; 632 | } 633 | } 634 | } 635 | 636 | /** Call this everytime a '\n' is encountered. */ 637 | void newline() { 638 | line++; 639 | lineStart = pos; 640 | } 641 | 642 | void unexpected(bool value, string message) 643 | { 644 | if (!value) 645 | { 646 | throw new DProtoSyntaxException(message, fileName, line + 1); 647 | } 648 | } 649 | 650 | /** Returns true if the name is a reserved word in D 651 | * 652 | * This will cause problems trying to use them as variables 653 | * Note: Some keywords are specifically whitelisted, 654 | * in order to allow usage of the protobuf names 655 | */ 656 | bool isDKeyword(string name) 657 | { 658 | // dfmt off 659 | enum KEYWORDS = [ 660 | "abstract", "alias", "align", "asm", "assert", "auto", 661 | "body", /+ "bool", +/ "break", "byte", 662 | "case", "cast", "catch", "cdouble", "cent", "cfloat", "char", "class", "const", "continue", "creal", 663 | "dchar", "debug", "default", "delegate", "delete", "deprecated", "do", /+ "double", +/ 664 | "else", "enum", "export", "extern", 665 | "false", "final", "finally", /+ "float", +/ "for", "foreach", "foreach_reverse", "function", 666 | "goto", 667 | "idouble", "if", "ifloat", "immutable", "import", "in", "inout", "int", "interface", "invariant", "ireal", "is", 668 | "lazy", "long", 669 | "macro", "mixin", "module", 670 | "new", "nothrow", "null", 671 | "out", "override", 672 | "package", "pragma", "private", "protected", "public", "pure", 673 | "real", "ref", "return", 674 | "scope", "shared", "short", "static", "struct", "super", "switch", "synchronized", 675 | "template", "this", "throw", "true", "try", "typedef", "typeid", "typeof", 676 | "ubyte", "ucent", "uint", "ulong", "union", "unittest", "ushort", 677 | "version", "void", "volatile", 678 | "wchar", "while", "with", 679 | "__FILE__", "__MODULE__", "__LINE__", "__FUNCTION__", "__PRETTY_FUNCTION__", 680 | "__gshared", "__traits", "__vector", "__parameters", 681 | ]; 682 | // dfmt on 683 | return KEYWORDS.canFind(name); 684 | } 685 | 686 | } 687 | 688 | return ProtoSchemaParser(name_, data_).readProtoPackage(); 689 | 690 | } 691 | 692 | -------------------------------------------------------------------------------- /import/dproto/serialize.d: -------------------------------------------------------------------------------- 1 | /******************************************************************************* 2 | * Serialization/deserialization code 3 | * 4 | * Author: Matthew Soucy, dproto@msoucy.me 5 | */ 6 | module dproto.serialize; 7 | 8 | import dproto.exception; 9 | import dproto.compat; 10 | 11 | import std.algorithm; 12 | import std.array; 13 | import std.conv; 14 | import std.exception; 15 | import std.range; 16 | import std.system : Endian; 17 | import std.traits; 18 | 19 | /******************************************************************************* 20 | * Returns whether the given string is a protocol buffer primitive 21 | * 22 | * Params: 23 | * type = The type to check for 24 | * Returns: True if the type is a protocol buffer primitive 25 | */ 26 | bool isBuiltinType(string type) @safe pure nothrow { 27 | return ["int32" , "sint32", "int64", "sint64", "uint32", "uint64", "bool", 28 | "fixed64", "sfixed64", "double", "bytes", "string", 29 | "fixed32", "sfixed32", "float"].canFind(type); 30 | } 31 | 32 | unittest { 33 | assert(isBuiltinType("sfixed32") == true); 34 | assert(isBuiltinType("double") == true); 35 | assert(isBuiltinType("string") == true); 36 | assert(isBuiltinType("int128") == false); 37 | assert(isBuiltinType("quad") == false); 38 | } 39 | 40 | template PossiblyNullable(T) { 41 | static if(is(T == enum)) { 42 | alias PossiblyNullable = T; 43 | } else { 44 | import std.typecons : Nullable; 45 | alias PossiblyNullable = Nullable!T; 46 | } 47 | } 48 | 49 | template UnspecifiedDefaultValue(T) { 50 | static if(is(T == enum)) { 51 | import std.traits : EnumMembers; 52 | enum UnspecifiedDefaultValue = EnumMembers!(T)[0]; 53 | } else { 54 | enum UnspecifiedDefaultValue = T.init; 55 | } 56 | } 57 | 58 | template SpecifiedDefaultValue(T, string value) { 59 | import std.conv : to; 60 | enum SpecifiedDefaultValue = to!T(value); 61 | } 62 | 63 | /******************************************************************************* 64 | * Maps the given type string to the data type it represents 65 | */ 66 | template BuffType(string T) { 67 | // Msg type 0 68 | static if(T == "int32" || T == "sint32") alias BuffType = int; 69 | else static if(T == "int64" || T == "sint64") alias BuffType = long; 70 | else static if(T == "uint32") alias BuffType = uint; 71 | else static if(T == "uint64") alias BuffType = ulong; 72 | else static if(T == "bool") alias BuffType = bool; 73 | // Msg type 1 74 | else static if(T == "fixed64") alias BuffType = ulong; 75 | else static if(T == "sfixed64") alias BuffType = long; 76 | else static if(T == "double") alias BuffType = double; 77 | // Msg type 2 78 | else static if(T == "bytes") alias BuffType = ubyte[]; 79 | else static if(T == "string") alias BuffType = string; 80 | // Msg type 3,4 deprecated. Will not support. 81 | // Msg type 5 82 | else static if(T == "fixed32") alias BuffType = uint; 83 | else static if(T == "sfixed32") alias BuffType = int; 84 | else static if(T == "float") alias BuffType = float; 85 | } 86 | 87 | unittest { 88 | assert(is(BuffType!"sfixed32" == int) == true); 89 | assert(is(BuffType!"double" == double) == true); 90 | assert(is(BuffType!"string" == string) == true); 91 | assert(is(BuffType!"bytes" : const ubyte[]) == true); 92 | assert(is(BuffType!"sfixed64" == int) == false); 93 | } 94 | 95 | /******************************************************************************* 96 | * Removes bytes from the range as if it were read in 97 | * 98 | * Params: 99 | * header = The data header 100 | * data = The data to read from 101 | */ 102 | void defaultDecode(R)(ulong header, ref R data) 103 | if(isInputRange!R && is(ElementType!R : const ubyte)) 104 | { 105 | switch(header.wireType) { 106 | case 0: 107 | data.readProto!"int32"(); 108 | break; 109 | case 1: 110 | data.readProto!"fixed64"(); 111 | break; 112 | case 2: 113 | data.readProto!"bytes"(); 114 | break; 115 | case 5: 116 | data.readProto!"fixed32"(); 117 | break; 118 | default: 119 | break; 120 | } 121 | } 122 | 123 | /******************************************************************************* 124 | * Maps the given type string to the wire type number 125 | */ 126 | @nogc 127 | auto msgType(string T) pure nothrow @safe { 128 | switch(T) { 129 | case "int32", "sint32", "uint32": 130 | case "int64", "sint64", "uint64": 131 | case "bool": 132 | return 0; 133 | case "fixed64", "sfixed64", "double": 134 | return 1; 135 | case "bytes", "string": 136 | return 2; 137 | case "fixed32", "sfixed32", "float": 138 | return 5; 139 | default: 140 | return 2; 141 | } 142 | } 143 | 144 | /******************************************************************************* 145 | * Encodes a number in its zigzag encoding 146 | * 147 | * Params: 148 | * src = The raw integer to encode 149 | * Returns: The zigzag-encoded value 150 | */ 151 | @nogc Unsigned!T toZigZag(T)(T src) pure nothrow @safe @property 152 | if(isIntegral!T && isSigned!T) 153 | { 154 | T ret = (src << 1) ^ (src >> (T.sizeof * 8 - 1)); 155 | 156 | return cast(Unsigned!T) ret; 157 | } 158 | 159 | unittest { 160 | assert(0.toZigZag() == 0); 161 | assert((-1).toZigZag() == 1); 162 | assert(1.toZigZag() == 2); 163 | assert((-2).toZigZag() == 3); 164 | assert(2147483647.toZigZag() == 4294967294); 165 | assert((-2147483648).toZigZag() == 4294967295); 166 | } 167 | 168 | /******************************************************************************* 169 | * Decodes a number from its zigzag encoding 170 | * 171 | * Params: 172 | * src = The zigzag-encoded value to decode 173 | * Returns: The raw integer 174 | */ 175 | @nogc Signed!T fromZigZag(T)(T src) pure nothrow @safe @property 176 | if(isIntegral!T && isUnsigned!T) 177 | { 178 | return (src >>> 1) ^ -(src & 1); 179 | } 180 | 181 | unittest { 182 | assert(0U.fromZigZag() == 0); 183 | assert(1U.fromZigZag() == -1); 184 | assert(2U.fromZigZag() == 1); 185 | assert(3U.fromZigZag() == -2); 186 | assert(4294967294U.fromZigZag() == 2147483647); 187 | assert(4294967295U.fromZigZag() == -2147483648); 188 | 189 | foreach(i;-3..3){ 190 | assert(i.toZigZag.fromZigZag == i); 191 | long i2=i; 192 | assert(i2.toZigZag.fromZigZag == i2); 193 | } 194 | } 195 | 196 | /******************************************************************************* 197 | * Get the wire type from the encoding value 198 | * 199 | * Params: 200 | * data = The data header 201 | * Returns: The wire type value 202 | */ 203 | @nogc ubyte wireType(ulong data) @safe @property pure nothrow { 204 | return data&7; 205 | } 206 | 207 | unittest { 208 | assert((0x08).wireType() == 0); // Test for varints 209 | assert((0x09).wireType() == 1); // Test 64-bit 210 | assert((0x12).wireType() == 2); // Test length-delimited 211 | } 212 | 213 | /******************************************************************************* 214 | * Get the message number from the encoding value 215 | * 216 | * Params: 217 | * data = The data header 218 | * Returns: The message number 219 | */ 220 | @nogc ulong msgNum(ulong data) @safe @property pure nothrow { 221 | return data>>3; 222 | } 223 | 224 | unittest { 225 | assert((0x08).msgNum() == 1); 226 | assert((0x11).msgNum() == 2); 227 | assert((0x1a).msgNum() == 3); 228 | assert((0x22).msgNum() == 4); 229 | } 230 | 231 | /******************************************************************************* 232 | * Read a VarInt-encoded value from a data stream 233 | * 234 | * Removes the bytes that represent the data from the stream 235 | * 236 | * Params: 237 | * src = The data stream 238 | * Returns: The decoded value 239 | */ 240 | T readVarint(T = ulong, R)(auto ref R src) 241 | if(isInputRange!R && is(ElementType!R : const ubyte)) 242 | { 243 | auto i = src.countUntil!( a=>!(a&0x80) )() + 1; 244 | auto ret = src.take(i); 245 | src.popFrontExactly(i); 246 | return ret.fromVarint!T(); 247 | } 248 | 249 | /******************************************************************************* 250 | * Encode an unsigned value into a VarInt-encoded series of bytes 251 | * 252 | * Params: 253 | * r = output range 254 | * src = The value to encode 255 | * Returns: The created VarInt 256 | */ 257 | void toVarint(R, T)(ref R r, T src) @safe @property 258 | if(isOutputRange!(R, ubyte) && isIntegral!T && isUnsigned!T) 259 | { 260 | immutable ubyte maxMask = 0b_1000_0000; 261 | 262 | while( src >= maxMask ) 263 | { 264 | r.put(cast(ubyte)(src | maxMask)); 265 | src >>= 7; 266 | } 267 | 268 | r.put(cast(ubyte) src); 269 | } 270 | 271 | /******************************************************************************* 272 | * Encode a signed value into a VarInt-encoded series of bytes 273 | * 274 | * This function is useful for encode int32 and int64 value types 275 | * (Do not confuse it with signed values encoded by ZigZag!) 276 | * 277 | * Params: 278 | * r = output range 279 | * src = The value to encode 280 | * Returns: The created VarInt 281 | */ 282 | void toVarint(R)(ref R r, long src) @safe @property 283 | if(isOutputRange!(R, ubyte)) 284 | { 285 | ulong u = src; 286 | toVarint(r, u); 287 | } 288 | 289 | unittest { 290 | static ubyte[] toVarint(ulong val) @property 291 | { 292 | auto r = appender!(ubyte[])(); 293 | .toVarint(r, val); 294 | return r.data; 295 | } 296 | assert(equal(toVarint(150), [0x96, 0x01])); 297 | assert(equal(toVarint(3), [0x03])); 298 | assert(equal(toVarint(270), [0x8E, 0x02])); 299 | assert(equal(toVarint(86942), [0x9E, 0xA7, 0x05])); 300 | assert(equal(toVarint(ubyte.max), [0xFF, 0x01])); 301 | assert(equal(toVarint(uint.max), [0xFF, 0xFF, 0xFF, 0xFF, 0xF])); 302 | assert(equal(toVarint(ulong.max), [0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01])); 303 | assert(equal(toVarint(-1), [0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01])); 304 | assert(toVarint(-12345).fromVarint!int == -12345); 305 | assert(toVarint(int.min).fromVarint!int == int.min); 306 | } 307 | 308 | /******************************************************************************* 309 | * Decode a VarInt-encoded series of bytes into an unsigned value 310 | * 311 | * Params: 312 | * src = The data stream 313 | * Returns: The decoded value 314 | */ 315 | T fromVarint(T = ulong, R)(R src) @property 316 | if(isInputRange!R && is(ElementType!R : const ubyte) && 317 | isIntegral!T && isUnsigned!T) 318 | { 319 | immutable ubyte mask = 0b_0111_1111; 320 | T ret; 321 | 322 | size_t offset; 323 | foreach(val; src) 324 | { 325 | ret |= cast(T)(val & mask) << offset; 326 | 327 | enforce( 328 | offset < T.sizeof * 8, 329 | "Varint value is too big for the type " ~ T.stringof 330 | ); 331 | 332 | offset += 7; 333 | } 334 | 335 | return ret; 336 | } 337 | 338 | /******************************************************************************* 339 | * Decode a VarInt-encoded series of bytes into a signed value 340 | * 341 | * Params: 342 | * src = The data stream 343 | * Returns: The decoded value 344 | */ 345 | T fromVarint(T, R)(R src) @property 346 | if(isInputRange!R && is(ElementType!R : const ubyte) && 347 | isIntegral!T && isSigned!T) 348 | { 349 | long r = fromVarint!ulong(src); 350 | return r.to!T; 351 | } 352 | 353 | unittest { 354 | ubyte[] ubs(ubyte[] vals...) { 355 | return vals.dup; 356 | } 357 | 358 | assert(ubs(0x96, 0x01).fromVarint() == 150); 359 | assert(ubs(0x03).fromVarint() == 3); 360 | assert(ubs(0x8E, 0x02).fromVarint() == 270); 361 | assert(ubs(0x9E, 0xA7, 0x05).fromVarint() == 86942); 362 | assert(ubs(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01).fromVarint!int() == -1); 363 | 364 | bool overflow = false; 365 | try 366 | ubs(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01).fromVarint(); 367 | catch(Exception) 368 | overflow = true; 369 | finally 370 | assert(overflow); 371 | } 372 | 373 | /// The type to encode an enum as 374 | enum ENUM_SERIALIZATION = "int32"; 375 | /// The message type to encode a packed message as 376 | enum PACKED_MSG_TYPE = 2; 377 | 378 | /******************************************************************************* 379 | * Test a range for being a valid ProtoBuf input range 380 | * 381 | * Params: 382 | * R = type to test 383 | * Returns: The value 384 | */ 385 | 386 | enum isProtoInputRange(R) = isInputRange!R && is(ElementType!R : const ubyte); 387 | 388 | /******************************************************************************* 389 | * Decode a series of bytes into a value 390 | * 391 | * Params: 392 | * src = The data stream 393 | * Returns: The decoded value 394 | */ 395 | BuffType!T readProto(string T, R)(auto ref R src) 396 | if(isProtoInputRange!R && T.msgType == "int32".msgType) 397 | { 398 | alias BT = BuffType!T; 399 | static if(T == "sint32" || T == "sint64") 400 | return src.readVarint!(Unsigned!BT).fromZigZag; 401 | else static if(T == "bool") 402 | return src.readVarint.to!BT; 403 | else 404 | return src.readVarint!BT; 405 | } 406 | 407 | /// Ditto 408 | BuffType!T readProto(string T, R)(auto ref R src) 409 | if(isProtoInputRange!R && 410 | (T.msgType == "double".msgType || T.msgType == "float".msgType)) 411 | { 412 | import std.bitmanip : read, Endian; 413 | return src.read!(BuffType!T, Endian.littleEndian)(); 414 | } 415 | 416 | /// Ditto 417 | BuffType!T readProto(string T, R)(auto ref R src) 418 | if(isProtoInputRange!R && T.msgType == "string".msgType) 419 | { 420 | BuffType!T ret; 421 | auto len = src.readProto!"uint32"(); 422 | ret.reserve(len); 423 | foreach(i; 0..len) { 424 | ret ~= src.front; 425 | src.popFront(); 426 | } 427 | return ret; 428 | } 429 | 430 | /******************************************************************************* 431 | * Test a range for being a valid ProtoBuf output range 432 | * 433 | * Params: 434 | * R = type to test 435 | * Returns: The value 436 | */ 437 | 438 | enum isProtoOutputRange(R) = isOutputRange!(R, ubyte); 439 | 440 | /******************************************************************************* 441 | * Encode a value into a series of bytes 442 | * 443 | * Params: 444 | * r = output range 445 | * src = The raw data 446 | * Returns: The encoded value 447 | */ 448 | void writeProto(string T, R)(ref R r, BuffType!T src) 449 | if(isProtoOutputRange!R && T.msgType == "int32".msgType) 450 | { 451 | static if(T == "sint32" || T == "sint64"){ 452 | toVarint(r, src.toZigZag); 453 | } else{ 454 | toVarint(r, src); 455 | } 456 | } 457 | 458 | /// Ditto 459 | void writeProto(string T, R)(ref R r, BuffType!T src) 460 | if(isProtoOutputRange!R && 461 | (T.msgType == "double".msgType || T.msgType == "float".msgType)) 462 | { 463 | import std.bitmanip : nativeToLittleEndian; 464 | r.put(src.nativeToLittleEndian!(BuffType!T)[]); 465 | } 466 | 467 | /// Ditto 468 | void writeProto(string T, R)(ref R r, const BuffType!T src) 469 | if(isProtoOutputRange!R && T.msgType == "string".msgType) 470 | { 471 | toVarint(r, src.length); 472 | r.put(cast(ubyte[])src); 473 | } 474 | 475 | // Unit test for issue #115 476 | unittest 477 | { 478 | static if (__traits(compiles, {import std.meta : AliasSeq;})) { 479 | import std.meta : AliasSeq; 480 | } else { 481 | import std.typetuple : TypeTuple; 482 | alias AliasSeq = TypeTuple; 483 | } 484 | 485 | for(int counter=0;counter<2;counter++){ 486 | foreach (T; AliasSeq!("bool", "int32", "uint32", "fixed32", "int64", "uint64", "fixed64", "sfixed32", "sfixed64", "sint64", "sint32")) { 487 | alias T2 = BuffType!T; 488 | auto r = appender!(ubyte[])(); 489 | static if (is(T2 == bool)) 490 | T2 src = counter==0 ? false : true; 491 | else 492 | T2 src = counter==0 ? -1 : 5; 493 | r.writeProto!T(src); 494 | 495 | T2 src2 = readProto!T(r.data); 496 | import std.conv:text; 497 | assert(src == src2, text("error: ", T.stringof, " ", src2, " ", src)); 498 | } 499 | } 500 | } 501 | 502 | /******************************************************************************* 503 | * Simple range that ignores data but counts the length 504 | */ 505 | struct CntRange 506 | { 507 | @nogc: 508 | size_t cnt; 509 | void put(in ubyte) @safe { ++cnt; } 510 | void put(in ubyte[] ary) @safe { cnt += ary.length; } 511 | alias cnt this; 512 | } 513 | -------------------------------------------------------------------------------- /import/dproto/unittests.d: -------------------------------------------------------------------------------- 1 | /******************************************************************************* 2 | * Library unit test file 3 | * 4 | * Authors: Matthew Soucy, dproto@msoucy.me 5 | */ 6 | module dproto.unittests; 7 | 8 | import dproto.dproto; 9 | 10 | unittest 11 | { 12 | assert(__traits(compiles, ProtocolBufferFromString!"message Test 13 | { 14 | optional string verySimple = 1; 15 | }")); 16 | } 17 | 18 | unittest 19 | { 20 | assert(__traits(compiles, ProtocolBufferFromString!" 21 | message Test 22 | { 23 | optional string verySimple = 1; 24 | enum TestEnum 25 | { 26 | ONE = 1; 27 | UNO = 1; 28 | TOW = 2; 29 | } 30 | }")); 31 | } 32 | 33 | unittest 34 | { 35 | mixin ProtocolBufferFromString!" 36 | message Test 37 | { 38 | required int32 id = 1; 39 | optional string verySimple = 2; 40 | enum TestEnum 41 | { 42 | ONE = 1; 43 | UNO = 1; 44 | TOW = 2; 45 | } 46 | optional TestEnum testValue = 3; 47 | }"; 48 | } 49 | 50 | unittest 51 | { 52 | assert(__traits(compiles, ProtocolBufferFromString!"message Test 53 | { 54 | optional string verySimple = 1; 55 | enum TestEnum 56 | { 57 | ONE = 1; 58 | UNO = 1; 59 | TOW = 2; 60 | } 61 | 62 | optional string testValue = 2; 63 | }")); 64 | } 65 | 66 | unittest 67 | { 68 | assert(__traits(compiles, ProtocolBufferFromString!" 69 | message Test 70 | { 71 | optional string verySimple = 1; 72 | message NestedTest 73 | { 74 | optional string verySimple = 1; 75 | } 76 | 77 | optional NestedTest value = 2; 78 | }")); 79 | } 80 | 81 | unittest 82 | { 83 | assert(__traits(compiles, ProtocolBufferFromString!" 84 | message Test 85 | { 86 | optional string verySimple = 1; 87 | message NestedTest 88 | { 89 | optional string verySimple2 = 1; 90 | } 91 | 92 | optional NestedTest value = 2; 93 | }")); 94 | } 95 | 96 | unittest 97 | { 98 | assert(__traits(compiles, ProtocolBufferFromString!" 99 | message Test 100 | { 101 | optional string verySimple = 1; 102 | message NestedTest 103 | { 104 | optional string verySimple = 1; 105 | } 106 | 107 | repeated NestedTest value = 2; 108 | }")); 109 | } 110 | 111 | unittest 112 | { 113 | assert(__traits(compiles, ProtocolBufferFromString!" 114 | message Test 115 | { 116 | required int32 id = 3; 117 | optional string verySimple = 1; 118 | message NestedTest 119 | { 120 | required string verySimple = 1; 121 | } 122 | 123 | required NestedTest value = 2; 124 | }")); 125 | } 126 | 127 | unittest 128 | { 129 | assert(__traits(compiles, ProtocolBufferFromString!" 130 | message Test 131 | { 132 | required int32 id = 3; 133 | optional string verySimple = 1; 134 | message NestedTest 135 | { 136 | required string verySimple = 1; 137 | } 138 | 139 | repeated NestedTest value = 2; 140 | }")); 141 | } 142 | 143 | unittest 144 | { 145 | assert(__traits(compiles, ProtocolBufferFromString!" 146 | message Test 147 | { 148 | required int32 id = 3; 149 | optional string verySimple = 1; 150 | message NestedTest 151 | { 152 | required string verySimple = 1; 153 | } 154 | 155 | optional NestedTest value = 2; 156 | }")); 157 | } 158 | 159 | unittest 160 | { 161 | assert(__traits(compiles, ProtocolBufferFromString!" 162 | message Test 163 | { 164 | required int32 id = 3; 165 | optional string verySimple = 1; 166 | message NestedTest 167 | { 168 | required string verySimple = 1; 169 | } 170 | 171 | repeated NestedTest value = 2; 172 | }")); 173 | } 174 | 175 | unittest 176 | { 177 | enum serviceDefinition = " 178 | message ServiceRequest { 179 | string request = 1; 180 | } 181 | message ServiceResponse { 182 | string response = 1; 183 | } 184 | service TestService { 185 | rpc TestMethod (ServiceRequest) returns (ServiceResponse); 186 | } 187 | "; 188 | 189 | // Force code coverage in doveralls 190 | import std.string; 191 | import std.format; 192 | import dproto.parse; 193 | 194 | auto normalizedServiceDefinition = "%3.3p".format(ParseProtoSchema("", serviceDefinition)); 195 | 196 | assert(__traits(compiles, ProtocolBufferFromString!serviceDefinition)); 197 | assert(__traits(compiles, ProtocolBufferInterface!serviceDefinition)); 198 | assert(__traits(compiles, ProtocolBufferRpc!serviceDefinition)); 199 | assert(__traits(compiles, ProtocolBufferImpl!serviceDefinition)); 200 | assert(__traits(compiles, ProtocolBufferStruct!serviceDefinition)); 201 | 202 | // Example from README.md. 203 | mixin ProtocolBufferInterface!serviceDefinition; 204 | 205 | class ServiceImplementation : TestService 206 | { 207 | ServiceResponse TestMethod(ServiceRequest input) 208 | { 209 | ServiceResponse output; 210 | output.response = "received: " ~ input.request; 211 | return output; 212 | } 213 | } 214 | 215 | auto serviceTest = new ServiceImplementation; 216 | ServiceRequest input; 217 | input.request = "message"; 218 | assert(serviceTest.TestMethod(input).response == "received: message"); 219 | 220 | } 221 | 222 | unittest 223 | { 224 | mixin ProtocolBufferFromString!" 225 | enum PhoneType { 226 | MOBILE = 0; 227 | HOME = 0xf; 228 | WORK = 071; 229 | } 230 | "; 231 | 232 | assert(PhoneType.MOBILE == 0); 233 | assert(PhoneType.HOME == 15); 234 | assert(PhoneType.WORK == 57); 235 | } 236 | 237 | unittest 238 | { 239 | mixin ProtocolBufferFromString!" 240 | enum PhoneType { 241 | MOBILE = 0; 242 | HOME = 0; 243 | WORK = 2; 244 | } 245 | 246 | message Person { 247 | required string name = 1; 248 | required int32 id = 2; 249 | optional string email = 3; 250 | 251 | message PhoneNumber { 252 | required string number = 1; 253 | optional PhoneType type = 2 [default = HOME]; 254 | } 255 | 256 | repeated PhoneNumber phone = 4; 257 | } 258 | "; 259 | 260 | Person t; 261 | assert(t.name == ""); 262 | assert(t.id == 0); 263 | assert(t.phone.length == 0); 264 | version (Have_painlessjson) 265 | { 266 | assert(t.toJson() == `{"email":"","id":0,"name":"","phone":[]}`); 267 | } 268 | 269 | t.name = "Max Musterman"; 270 | assert(t.name == "Max Musterman"); 271 | 272 | t.id = 3; 273 | assert(t.id == 3); 274 | 275 | t.email = "Max.Musterman@example.com"; 276 | assert(t.email); 277 | assert(t.email == "Max.Musterman@example.com"); 278 | 279 | Person.PhoneNumber pn1; 280 | pn1.number = "0123456789"; 281 | assert(pn1.number == "0123456789"); 282 | assert(pn1.type == PhoneType.HOME); 283 | assert(pn1.type == PhoneType.MOBILE); 284 | 285 | pn1.type = PhoneType.WORK; 286 | assert(pn1.type == PhoneType.WORK); 287 | assert(pn1.type); 288 | assert(pn1.type == 2); 289 | 290 | t.phone ~= pn1; 291 | assert(t.phone[0] == pn1); 292 | assert(t.phone.length == 1); 293 | 294 | version (Have_painlessjson) 295 | { 296 | assert( 297 | t.toJson() == `{"email":"Max.Musterman@example.com","id":3,"name":"Max Musterman","phone":[{"number":"0123456789","type":2}]}`); 298 | } 299 | 300 | pn1.type = pn1.type.init; 301 | assert(pn1.type == PhoneType.HOME); 302 | 303 | t.phone = t.phone.init; 304 | assert(t.phone.length == 0); 305 | 306 | t.email = t.email.init; 307 | assert(t.email == ""); 308 | } 309 | 310 | unittest 311 | { 312 | mixin ProtocolBufferFromString!" 313 | message Person { 314 | required string name = 1; 315 | required int32 id = 2; 316 | optional string email = 3; 317 | 318 | enum PhoneType { 319 | MOBILE = 0; 320 | HOME = 0; 321 | WORK = 2; 322 | } 323 | 324 | message PhoneNumber { 325 | required string number = 1; 326 | optional PhoneType type = 2 [default = HOME]; 327 | } 328 | 329 | repeated PhoneNumber phone = 4; 330 | } 331 | 332 | message AddressBook { 333 | repeated Person person = 1; 334 | } 335 | "; 336 | 337 | Person t; 338 | assert(t.name == ""); 339 | assert(t.id == 0); 340 | assert(t.phone.length == 0); 341 | 342 | t.name = "Max Musterman"; 343 | assert(t.name == "Max Musterman"); 344 | 345 | t.id = 3; 346 | assert(t.id == 3); 347 | 348 | t.email = "Max.Musterman@example.com"; 349 | assert(t.email); 350 | assert(t.email == "Max.Musterman@example.com"); 351 | 352 | Person.PhoneNumber pn1; 353 | pn1.number = "0123456789"; 354 | assert(pn1.number == "0123456789"); 355 | 356 | t.phone ~= pn1; 357 | assert(t.phone[0] == pn1); 358 | assert(t.phone.length == 1); 359 | 360 | t.phone = t.phone.init; 361 | assert(t.phone.length == 0); 362 | 363 | t.email = t.email.init; 364 | assert(t.email == ""); 365 | 366 | AddressBook addressbook; 367 | assert(addressbook.person.length == 0); 368 | addressbook.person ~= t; 369 | addressbook.person ~= t; 370 | assert(addressbook.person[0] == t); 371 | assert(addressbook.person[0] == addressbook.person[1]); 372 | assert(addressbook.person.length == 2); 373 | } 374 | 375 | unittest 376 | { 377 | mixin ProtocolBufferFromString!" 378 | enum PhoneType { 379 | MOBILE = 0; 380 | HOME = 0; 381 | WORK = 2; 382 | } 383 | 384 | message Person { 385 | required string name = 1; 386 | required int32 id = 2; 387 | optional string email = 3; 388 | 389 | message PhoneNumber { 390 | required string number = 1; 391 | optional PhoneType type = 2 [default = HOME]; 392 | } 393 | 394 | repeated PhoneNumber phone = 4; 395 | } 396 | 397 | message AddressBook { 398 | repeated Person person = 1; 399 | } 400 | "; 401 | 402 | Person t; 403 | t.name = "Max Musterman"; 404 | t.id = 3; 405 | t.email = "test@example.com"; 406 | 407 | Person.PhoneNumber pn1; 408 | pn1.number = "0123456789"; 409 | pn1.type = PhoneType.WORK; 410 | 411 | Person.PhoneNumber pn2; 412 | pn2.number = "0123456789"; 413 | 414 | t.phone = [pn1, pn2]; 415 | AddressBook addressbook; 416 | addressbook.person ~= t; 417 | addressbook.person ~= t; 418 | 419 | ubyte[] serializedObject = addressbook.serialize(); 420 | 421 | AddressBook addressbook2 = AddressBook.fromProto(serializedObject); 422 | assert(addressbook2.person.length == 2); 423 | foreach (t2; addressbook2.person[0 .. 1]) 424 | { 425 | assert(t2.name == "Max Musterman"); 426 | assert(t2.id == 3); 427 | assert(t2.email); 428 | assert(t2.email == "test@example.com"); 429 | assert(t2.phone[0].number == "0123456789"); 430 | assert(t2.phone[0].type == PhoneType.WORK); 431 | assert(t2.phone[1].number == "0123456789"); 432 | assert(t2.phone[1].type == PhoneType.HOME); 433 | assert(t2.phone[1].type == PhoneType.MOBILE); 434 | assert(t2.phone.length == 2); 435 | } 436 | //the gdc-4.8 evaluates false here. Maybe an compiler bug. 437 | version (DigitalMars) 438 | { 439 | assert(addressbook2.person[0] == addressbook.person[1]); 440 | } 441 | } 442 | 443 | unittest 444 | { 445 | mixin ProtocolBufferFromString!" 446 | message Person { 447 | required string name = 1; 448 | required int32 id = 2; 449 | optional string email = 3; 450 | 451 | enum PhoneType { 452 | MOBILE = 0; 453 | HOME = 0; 454 | WORK = 2; 455 | } 456 | 457 | message PhoneNumber { 458 | required string number = 1; 459 | optional PhoneType type = 2 [default = HOME]; 460 | } 461 | 462 | repeated PhoneNumber phone = 4; 463 | } 464 | 465 | message AddressBook { 466 | repeated Person person = 1; 467 | } 468 | "; 469 | 470 | Person t; 471 | assert(t.name == ""); 472 | assert(t.id == 0); 473 | assert(t.phone.length == 0); 474 | 475 | t.name = "Max Musterman"; 476 | assert(t.name == "Max Musterman"); 477 | 478 | t.id = 3; 479 | assert(t.id == 3); 480 | 481 | t.email = "Max.Musterman@example.com"; 482 | assert(t.email); 483 | assert(t.email == "Max.Musterman@example.com"); 484 | 485 | Person.PhoneNumber pn1; 486 | pn1.number = "0123456789"; 487 | assert(pn1.number == "0123456789"); 488 | assert(pn1.type == Person.PhoneType.HOME); 489 | assert(pn1.type == Person.PhoneType.MOBILE); 490 | 491 | pn1.type = Person.PhoneType.WORK; 492 | assert(pn1.type == Person.PhoneType.WORK); 493 | assert(pn1.type == 2); 494 | assert(pn1.type); 495 | 496 | t.phone ~= pn1; 497 | assert(t.phone[0] == pn1); 498 | assert(t.phone.length == 1); 499 | 500 | pn1.type = pn1.type.init; 501 | assert(pn1.type == Person.PhoneType.HOME); 502 | 503 | t.phone = t.phone.init; 504 | assert(t.phone.length == 0); 505 | 506 | t.email = t.email.init; 507 | assert(t.email == ""); 508 | 509 | AddressBook addressbook; 510 | assert(addressbook.person.length == 0); 511 | addressbook.person ~= t; 512 | addressbook.person ~= t; 513 | assert(addressbook.person[0] == t); 514 | assert(addressbook.person[0] == addressbook.person[1]); 515 | assert(addressbook.person.length == 2); 516 | 517 | static struct OutBuf 518 | { 519 | @nogc: 520 | @safe: 521 | void put(in ubyte) 522 | { 523 | } 524 | 525 | void put(in ubyte[]) 526 | { 527 | } 528 | } 529 | 530 | @nogc void testNoGC() 531 | { 532 | OutBuf buf; 533 | addressbook.serializeTo(buf); 534 | } 535 | 536 | testNoGC(); 537 | } 538 | 539 | unittest 540 | { 541 | mixin ProtocolBufferFromString!" 542 | message Person { 543 | required string name = 1; 544 | } 545 | "; 546 | 547 | static auto rvalue(in ubyte[] val) 548 | { 549 | return val; 550 | } 551 | 552 | enum data = cast(ubyte[])[1 << 3 | 2, "abc".length] ~ cast(ubyte[]) "abc"; 553 | const(ubyte)[] val = data; 554 | assert(val.length == 5); 555 | assert(Person(rvalue(val)).name == "abc"); 556 | assert(val.length == 5); 557 | assert(Person(val).name == "abc"); 558 | assert(val.length == 0); 559 | Person p; 560 | val = data; 561 | assert(val.length == 5); 562 | p.deserialize(rvalue(val)); 563 | assert(val.length == 5); 564 | assert(p.name == "abc"); 565 | p.name = null; 566 | p.deserialize(val); 567 | assert(val.length == 0); 568 | assert(p.name == "abc"); 569 | } 570 | 571 | unittest 572 | { 573 | mixin ProtocolBufferFromString!" 574 | message Field_Name_Equals_Internal_Variable_Name { 575 | required int32 r = 1; 576 | required int32 data = 2; 577 | required int32 msgdata = 3; 578 | } 579 | "; 580 | } 581 | 582 | unittest 583 | { 584 | import dproto.exception; 585 | import dproto.serialize; 586 | import dproto.parse; 587 | import std.string : strip; 588 | 589 | auto proto_src = `import "foo/baz.proto";`; 590 | auto proto_struct = ParseProtoSchema("", proto_src); 591 | auto d_src = proto_struct.toD; 592 | assert(`mixin ProtocolBuffer!"foo/baz.proto";` == d_src, 593 | "Mixin string should not have two double quotes " ~ d_src); 594 | assert(proto_src == proto_struct.toProto.strip, 595 | "Round tripping to protobuf source should yield starting text " ~ proto_struct.toProto); 596 | } 597 | 598 | unittest 599 | { 600 | mixin ProtocolBufferFromString!` 601 | enum RecordFlags 602 | { 603 | Announce = 1; 604 | Cancel = 2; 605 | SomeAnotherFlag = 4; // look at the enumeration! 606 | } 607 | message KeyValue 608 | { 609 | required bytes key = 1; 610 | optional RecordFlags flags = 2; 611 | optional bytes payload = 3; 612 | } 613 | message ECDSASignature 614 | { 615 | required bytes signature = 1; 616 | required bytes pubKey = 2; 617 | } 618 | message Signed 619 | { 620 | required ECDSASignature esignature = 1; 621 | required KeyValue keyValue = 2; 622 | }`; 623 | 624 | Signed d1; 625 | d1.keyValue.key = cast(ubyte[]) "key data"; 626 | d1.keyValue.payload = cast(ubyte[]) "value data"; 627 | auto ser = d1.serialize(); 628 | Signed d2 = ser; 629 | assert(d1.keyValue.key == d2.keyValue.key); 630 | assert(d1.keyValue.payload == d2.keyValue.payload); 631 | } 632 | 633 | unittest 634 | { 635 | 636 | mixin ProtocolBufferFromString!` 637 | message DNSPayload 638 | { 639 | repeated bytes assignOwnerPubKeys = 1; 640 | repeated bytes assignManagersPubKeys = 2; 641 | 642 | repeated bytes ns = 3; 643 | } 644 | `; 645 | 646 | DNSPayload p1; 647 | p1.ns ~= [1, 2, 3]; 648 | auto buf = p1.serialize(); 649 | 650 | DNSPayload p2; 651 | p2.deserialize(buf); 652 | assert(p1 == p2); 653 | } 654 | 655 | unittest 656 | { 657 | mixin ProtocolBufferFromString!" 658 | message Person { 659 | required uint32 id = 1; 660 | }"; 661 | } 662 | 663 | unittest 664 | { 665 | mixin ProtocolBufferFromString!` 666 | message TestStructure 667 | { 668 | optional string optional_string = 1; 669 | required string required_string = 2; 670 | repeated string repeated_string = 3; 671 | } 672 | `; 673 | import dproto.attributes : TagId; 674 | 675 | assert(TagId!(TestStructure.optional_string) == 1); 676 | assert(TagId!(TestStructure.required_string) == 2); 677 | assert(TagId!(TestStructure.repeated_string) == 3); 678 | } 679 | 680 | unittest 681 | { 682 | mixin ProtocolBufferFromString!" 683 | message Stats { 684 | optional int32 agility = 1; 685 | optional int32 stamina = 2; 686 | } 687 | message Character { 688 | optional string name = 1; 689 | optional Stats stats = 2; 690 | } 691 | message Account { 692 | optional string owner = 1; 693 | optional Character main = 2; 694 | } 695 | "; 696 | const int agility = 200; 697 | auto acct = Account(); 698 | auto main = Character(); 699 | main.name = "Hogan"; 700 | main.stats = Stats(); 701 | main.stats.get.agility = agility; 702 | acct.main = main; 703 | auto ser = acct.serialize(); 704 | Account acct_rx; 705 | acct_rx.deserialize(ser); 706 | import std.string : format; 707 | 708 | assert(acct_rx.main.get.stats.get.agility == agility, format("Expected %d, got %d", 709 | agility, acct_rx.main.get.stats.get.agility)); 710 | 711 | } 712 | 713 | unittest 714 | { 715 | enum pbstring = q{ 716 | enum Enum { 717 | A = 0; 718 | B = 1; 719 | C = 2; 720 | } 721 | 722 | message Msg { 723 | optional Enum unset = 1; 724 | optional Enum isset_first = 2 [default = A]; 725 | optional Enum isset_last = 3 [default = C]; 726 | required Enum unset_required = 4; 727 | required Enum isset_required = 5 [default = B]; 728 | optional int32 i1 = 6 [default = 42]; 729 | optional int32 i2 = 7; 730 | required int32 i3 = 8 [default = 24]; 731 | required int32 i4 = 9; 732 | } 733 | }; 734 | 735 | // Force code coverage in doveralls 736 | import std.string; 737 | import std.format; 738 | import dproto.parse; 739 | 740 | auto normalizedServiceDefinition = "%3.3p".format(ParseProtoSchema("", pbstring)); 741 | 742 | mixin ProtocolBufferFromString!pbstring; 743 | 744 | Msg msg; 745 | assert(msg.unset == Enum.A); 746 | assert(msg.isset_first == Enum.A); 747 | assert(msg.isset_last == Enum.C); 748 | assert(msg.unset_required == Enum.A); 749 | assert(msg.isset_required == Enum.B); 750 | assert(msg.i1 == 42); 751 | assert(msg.i2 == typeof(msg.i2).init); 752 | assert(msg.i3 == 24); 753 | assert(msg.i4 == typeof(msg.i4).init); 754 | } 755 | 756 | unittest 757 | { 758 | import dproto.parse; 759 | import dproto.exception; 760 | import std.exception; 761 | 762 | enum pbstring = q{ 763 | message Info { 764 | optional int32 version = 1 [default = -1]; 765 | } 766 | }; 767 | assertThrown!DProtoReservedWordException(ParseProtoSchema( 768 | "", 769 | `option dproto_reserved_fmt = "%s"; ` ~ pbstring)); 770 | assertNotThrown!DProtoReservedWordException(ParseProtoSchema( 771 | "", 772 | `option dproto_reserved_fmt = "%s_"; ` ~ pbstring)); 773 | assertNotThrown!DProtoReservedWordException(ParseProtoSchema( 774 | "", pbstring)); 775 | } 776 | 777 | unittest 778 | { 779 | mixin ProtocolBufferFromString!` 780 | message HeaderBBox { 781 | required sint64 left = 1; 782 | required sint64 right = 2; 783 | required sint64 top = 3; 784 | required sint64 bottom = 4; 785 | }`; 786 | HeaderBBox headerBBox; 787 | 788 | headerBBox.left = 10; 789 | headerBBox.right = 5; 790 | headerBBox.top = -32; 791 | headerBBox.bottom = -24; 792 | 793 | auto hbb = headerBBox.serialize(); 794 | headerBBox = HeaderBBox(hbb); // Error occurred here 795 | 796 | assert(headerBBox.left == 10); 797 | assert(headerBBox.right == 5); 798 | assert(headerBBox.top == -32); 799 | assert(headerBBox.bottom == -24); 800 | } 801 | 802 | unittest 803 | { 804 | assert(!__traits(compiles, mixin(`mixin ProtocolBufferFromString!q{ 805 | message One { 806 | required string a; 807 | required int32 b; 808 | } 809 | };`)), 810 | "Malformed proto structure accepted"); 811 | } 812 | 813 | unittest 814 | { 815 | import std.algorithm; 816 | 817 | mixin ProtocolBufferFromString!` 818 | message Foo { 819 | repeated uint32 arr = 1 [packed=true]; 820 | } 821 | `; 822 | 823 | Foo foo; 824 | foo.arr = [1]; 825 | 826 | auto serialized_foo = foo.serialize(); 827 | 828 | auto foo2 = Foo(serialized_foo); 829 | 830 | assert(equal(foo.arr, foo2.arr)); 831 | } 832 | 833 | unittest 834 | { 835 | // Issue #86 836 | import dproto.parse; 837 | import dproto.exception; 838 | enum pbstring = q{ 839 | message ReservedWordTest { 840 | required bool notReservedWord = 1; 841 | } 842 | }; 843 | mixin ProtocolBufferFromString!pbstring; 844 | assert(ParseProtoSchema("", pbstring).toD()); 845 | } 846 | 847 | unittest 848 | { 849 | import std.algorithm; 850 | 851 | mixin ProtocolBufferFromString!` 852 | message FooA { 853 | repeated uint32 arr = 1 [packed=true]; 854 | } 855 | message FooB { 856 | repeated uint32 arr = 1; 857 | } 858 | `; 859 | 860 | FooA foo; 861 | foo.arr = [1, 3, 5, 7, 2, 4, 6, 8]; 862 | 863 | auto serialized_foo = foo.serialize(); 864 | auto foo2 = FooB(serialized_foo); 865 | assert(equal(foo.arr, foo2.arr)); 866 | auto foo3 = FooA(foo2.serialize()); 867 | assert(equal(foo2.arr, foo3.arr)); 868 | } 869 | 870 | unittest 871 | { 872 | // Issue #86 873 | import dproto.parse; 874 | import dproto.exception; 875 | import std.exception; 876 | 877 | enum pbstring = q{ 878 | message ReservedWordTest { 879 | required bool notReservedWord; 880 | } 881 | }; 882 | assertThrown!DProtoSyntaxException(ParseProtoSchema("", pbstring)); 883 | } 884 | 885 | unittest 886 | { 887 | // Issue #89 888 | import dproto.dproto; 889 | 890 | mixin ProtocolBufferFromString!q{ 891 | message Test { 892 | repeated double id = 1 [packed = true]; 893 | } 894 | 895 | }; 896 | 897 | Test t; 898 | 899 | t.id = [123]; 900 | 901 | auto s = t.serialize(); 902 | t = Test(s); 903 | 904 | assert(t.id == [123]); 905 | } 906 | 907 | unittest 908 | { 909 | // Issue #92 910 | import dproto.dproto : ProtocolBufferFromString; 911 | import dproto.parse : ParseProtoSchema; 912 | 913 | enum syntaxProto2 = ` 914 | syntax = "proto2"; 915 | `; 916 | static assert(__traits(compiles, ProtocolBufferFromString!syntaxProto2)); 917 | 918 | enum schemaProto2 = ParseProtoSchema("", syntaxProto2); 919 | static assert(schemaProto2.syntax == `"proto2"`); 920 | 921 | enum syntaxProto3 = ` 922 | syntax = "proto3"; 923 | `; 924 | static assert(__traits(compiles, ProtocolBufferFromString!syntaxProto3)); 925 | 926 | enum schemaProto3 = ParseProtoSchema("", syntaxProto3); 927 | static assert(schemaProto3.syntax == `"proto3"`); 928 | 929 | enum syntaxNoEquals = ` 930 | syntax "proto2"; 931 | `; 932 | static assert(!__traits(compiles, ProtocolBufferFromString!syntaxNoEquals)); 933 | 934 | enum syntaxNoQuotes = ` 935 | syntax = proto2; 936 | `; 937 | static assert(!__traits(compiles, ProtocolBufferFromString!syntaxNoQuotes)); 938 | 939 | enum syntaxNoLQuote = ` 940 | syntax = proto2"; 941 | `; 942 | static assert(!__traits(compiles, ProtocolBufferFromString!syntaxNoLQuote)); 943 | 944 | enum syntaxNoRQuote = ` 945 | syntax = "proto2; 946 | `; 947 | static assert(!__traits(compiles, ProtocolBufferFromString!syntaxNoRQuote)); 948 | 949 | enum syntaxNoSemicolon = ` 950 | syntax = "proto2" 951 | `; 952 | static assert(!__traits(compiles, ProtocolBufferFromString!syntaxNoSemicolon)); 953 | } 954 | 955 | unittest 956 | { 957 | // Issue #26 958 | import dproto.parse; 959 | import dproto.exception; 960 | import std.exception; 961 | 962 | enum pbstring = q{ 963 | import public; 964 | }; 965 | mixin ProtocolBufferFromString!pbstring; 966 | } 967 | 968 | unittest 969 | { 970 | // Issue #26 971 | import dproto.parse; 972 | import dproto.exception; 973 | import std.exception; 974 | 975 | enum pbstring = q{ 976 | import public "proto/example.proto"; 977 | }; 978 | assert(ParseProtoSchema("", pbstring).toD()); 979 | } 980 | 981 | unittest 982 | { 983 | import dproto.parse; 984 | import dproto.exception; 985 | import std.exception; 986 | 987 | enum pbstring = q{ 988 | enum Foo { 989 | option allow_alias = false; 990 | ONE = 1; 991 | TWO = 1; 992 | THREE = 3; 993 | } 994 | }; 995 | assertThrown!DProtoSyntaxException(ParseProtoSchema("", pbstring)); 996 | 997 | enum pbstring2 = q{ 998 | enum Foo { 999 | ONE = 1; 1000 | TWO = 1; 1001 | THREE = 3; 1002 | } 1003 | }; 1004 | assertNotThrown!DProtoSyntaxException(ParseProtoSchema("", pbstring2)); 1005 | } 1006 | 1007 | unittest 1008 | { 1009 | // Issue #92 1010 | 1011 | import dproto.parse; 1012 | import dproto.exception; 1013 | import std.exception; 1014 | 1015 | enum pbstring = ` 1016 | syntax = "proto3"; 1017 | `; 1018 | assertNotThrown!DProtoSyntaxException(ParseProtoSchema("", pbstring)); 1019 | } 1020 | 1021 | unittest 1022 | { 1023 | enum pbstring = ` 1024 | message MyMsgAux { 1025 | optional string foo1=1; 1026 | } 1027 | 1028 | message MyMsg { 1029 | required MyMsgAux a1=1; 1030 | optional MyMsgAux a2=2; 1031 | } 1032 | `; 1033 | mixin ProtocolBufferFromString!pbstring; 1034 | 1035 | MyMsg proto; 1036 | proto.a1.foo1 = "bar"; // ok 1037 | proto.a2 = MyMsgAux(); 1038 | proto.a2.get.foo1 = "bar"; // Called `get' on null Nullable!MymsgAux 1039 | assert(proto.a1.foo1 == proto.a2.get.foo1); 1040 | } 1041 | 1042 | unittest 1043 | { 1044 | mixin ProtocolBufferFromString!" 1045 | message Person { 1046 | required string name = 1; 1047 | required int32 id = 2; 1048 | optional string email = 3; 1049 | 1050 | enum PhoneType { 1051 | MOBILE = 0; 1052 | HOME = 0; 1053 | WORK = 2; 1054 | } 1055 | 1056 | message PhoneNumber { 1057 | required string number = 1; 1058 | optional PhoneType type = 2 [default = HOME]; 1059 | } 1060 | 1061 | repeated PhoneNumber phone = 4; 1062 | } 1063 | "; 1064 | 1065 | Person t; 1066 | t.name = "Max Musterman"; 1067 | t.id = 3; 1068 | t.email = "Max.Musterman@example.com"; 1069 | 1070 | Person.PhoneNumber pn1; 1071 | pn1.number = "0123456789"; 1072 | pn1.type = Person.PhoneType.WORK; 1073 | t.phone ~= pn1; 1074 | 1075 | Person t2 = t; 1076 | t2.id = 5; 1077 | t2.email = "Namretsum.Xam@example.com"; 1078 | 1079 | Person.PhoneNumber pn2; 1080 | pn2.number = "9876543210"; 1081 | pn2.type = Person.PhoneType.HOME; 1082 | t2.phone = [pn2]; 1083 | 1084 | import std.stdio; 1085 | t.mergeFrom(t2); 1086 | assert(t.phone.length == 2); 1087 | assert(t.phone[0] == pn1); 1088 | assert(t.phone[1] == pn2); 1089 | assert(t.id == t.id); 1090 | assert(t.email == t2.email); 1091 | 1092 | } 1093 | 1094 | --------------------------------------------------------------------------------