├── test ├── proto │ ├── simple.proto │ ├── map.proto │ ├── import.proto │ ├── basic.proto │ ├── imported.proto │ ├── one_of.proto │ ├── nested_one_of.proto │ ├── wrappers.proto │ └── mumble.proto ├── test_helper.exs ├── protobuf │ ├── parse_test.exs │ ├── map_test.exs │ ├── delimited_test.exs │ ├── from_multiple_files_test.exs │ ├── nested_one_of_test.exs │ ├── encoder_test.exs │ ├── wrappers_test.exs │ ├── decoder_test.exs │ └── one_of_test.exs ├── utils │ ├── gpb_compile_helper.exs │ └── gpb_compile_helper_test.exs └── protobuf_test.exs ├── .formatter.exs ├── .gitignore ├── .travis.yml ├── bench ├── support │ └── proto.ex └── decode_bench.exs ├── lib ├── exprotobuf │ ├── one_of_field.ex │ ├── field.ex │ ├── config.ex │ ├── define_enum.ex │ ├── delimited.ex │ ├── parser.ex │ ├── utils.ex │ ├── encoder.ex │ ├── decoder.ex │ ├── builder.ex │ └── define_message.ex ├── serializable.ex └── exprotobuf.ex ├── .dialyzer.ignore-warnings ├── .dialyzer.ignore ├── mix.lock ├── mix.exs ├── imports_upgrade_guide.md ├── priv └── google_protobuf.proto ├── README.md └── LICENSE /test/proto/simple.proto: -------------------------------------------------------------------------------- 1 | message Basic { 2 | required uint32 f1 = 1; 3 | } 4 | -------------------------------------------------------------------------------- /.formatter.exs: -------------------------------------------------------------------------------- 1 | [ 2 | inputs: [ 3 | "lib/**/*.ex", 4 | "test/**/*.{ex, exs}", 5 | ], 6 | ] 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /_build 2 | /deps 3 | erl_crash.dump 4 | *.ez 5 | .exenv-version 6 | /doc 7 | /docs 8 | /tmp/ 9 | /bench/snapshots/ 10 | -------------------------------------------------------------------------------- /test/proto/map.proto: -------------------------------------------------------------------------------- 1 | message Value { 2 | required string value = 1; 3 | } 4 | 5 | message Entity { 6 | map properties = 1; 7 | } 8 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: elixir 3 | elixir: 4 | - 1.7 5 | - 1.8 6 | - 1.9 7 | otp_release: 8 | - 21.0 9 | - 22.0 10 | env: 11 | - MIX_ENV=test 12 | notifications: 13 | email: 14 | - paulschoenfelder@fastmail.com 15 | -------------------------------------------------------------------------------- /test/proto/import.proto: -------------------------------------------------------------------------------- 1 | package chat; 2 | 3 | option java_package = "com.appunite.chat"; 4 | 5 | import "imported.proto"; 6 | 7 | // From server to client 8 | message WebsocketServerContainer { 9 | required string authorization = 1; 10 | } 11 | -------------------------------------------------------------------------------- /test/test_helper.exs: -------------------------------------------------------------------------------- 1 | Code.require_file "./utils/gpb_compile_helper.exs", __DIR__ 2 | ExUnit.start 3 | 4 | defmodule Protobuf.Case do 5 | defmacro __using__(_) do 6 | quote do 7 | use ExUnit.Case, async: true 8 | alias GpbCompileHelper, as: Gpb 9 | end 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /bench/support/proto.ex: -------------------------------------------------------------------------------- 1 | defmodule Exprotobuf.Bench.Proto do 2 | use Protobuf, """ 3 | syntax="proto3"; 4 | 5 | package Demo.Data; 6 | 7 | message Request { 8 | 9 | } 10 | 11 | message Response { 12 | string name = 1; 13 | repeated string tags = 2; 14 | } 15 | """ 16 | end 17 | -------------------------------------------------------------------------------- /lib/exprotobuf/one_of_field.ex: -------------------------------------------------------------------------------- 1 | defmodule Protobuf.OneOfField do 2 | gpb_path = Path.join([Mix.Project.deps_path(), "gpb"]) 3 | headers_path = Path.join([gpb_path, "include", "gpb.hrl"]) 4 | 5 | @record Record.Extractor.extract(:gpb_oneof, from: headers_path) 6 | 7 | defstruct @record 8 | 9 | def record, do: @record 10 | end 11 | -------------------------------------------------------------------------------- /test/proto/basic.proto: -------------------------------------------------------------------------------- 1 | message Basic { 2 | enum Type { 3 | START = 1; 4 | STOP = 2; 5 | } 6 | 7 | required uint32 f1 = 1; 8 | required string f2 = 2; 9 | required Type type = 3; 10 | 11 | message AssocPair { 12 | optional string key = 1; 13 | optional string value = 2; 14 | } 15 | 16 | optional AssocPair args = 4; 17 | } 18 | -------------------------------------------------------------------------------- /test/protobuf/parse_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Protobuf.Parse.Test do 2 | use Protobuf.Case 3 | alias Protobuf.Parser 4 | 5 | test "parse string" do 6 | msg = "message Msg { required uint32 field1 = 1; }" 7 | [msg | _] = Parser.parse_string!("nofile", msg, []) 8 | assert is_tuple(msg) 9 | end 10 | 11 | test "raise exception with parse error" do 12 | assert_raise Parser.ParserError, fn -> 13 | Parser.parse_string!("nofile", "message ;", []) 14 | end 15 | end 16 | end 17 | -------------------------------------------------------------------------------- /lib/exprotobuf/field.ex: -------------------------------------------------------------------------------- 1 | defmodule Protobuf.Field do 2 | gpb_path = Path.join([Mix.Project.deps_path(), "gpb"]) 3 | headers_path = Path.join([gpb_path, "include", "gpb.hrl"]) 4 | 5 | case Version.compare(System.version(), "1.0.4") do 6 | :gt -> 7 | @record Record.Extractor.extract(:field, from: headers_path) 8 | 9 | _ -> 10 | @record Record.Extractor.extract(:"?gpb_field", from: headers_path) 11 | end 12 | 13 | defstruct @record 14 | 15 | def record, do: @record 16 | end 17 | -------------------------------------------------------------------------------- /lib/serializable.ex: -------------------------------------------------------------------------------- 1 | defprotocol Protobuf.Serializable do 2 | @moduledoc """ 3 | Defines the contract for serializing protobuf messages. 4 | """ 5 | @fallback_to_any true 6 | 7 | @doc """ 8 | Serializes the provided object as a protobuf message in binary form. 9 | """ 10 | def serialize(object) 11 | end 12 | 13 | defimpl Protobuf.Serializable, for: Any do 14 | def serialize(%{__struct__: module} = obj), do: module.encode(obj) 15 | def serialize(_), do: {:error, :not_serializable} 16 | end 17 | -------------------------------------------------------------------------------- /test/proto/imported.proto: -------------------------------------------------------------------------------- 1 | package authorization; 2 | 3 | option java_package = "com.appunite.chat"; 4 | 5 | // Message that is sent via server if authorization fail (sent via HTTP) 6 | message WrongAuthorizationHttpMessage { 7 | required string reason = 1; // reason why authorization fail 8 | } 9 | 10 | // Message that is sent via server if authorization fail 11 | message AuthorizationServerMessage { 12 | required string next_synchronization_token = 1; // token that should be sent to server next time synchronization occure 13 | } 14 | -------------------------------------------------------------------------------- /bench/decode_bench.exs: -------------------------------------------------------------------------------- 1 | defmodule Exprotobuf.DecodeBench do 2 | use Benchfella 3 | alias Exprotobuf.Bench.Proto.Request 4 | alias Exprotobuf.Bench.Proto.Response 5 | 6 | @request %Request{} 7 | |> Request.encode 8 | @response %Response{ 9 | name: "hello", 10 | tags: ["hello", "world"] 11 | } 12 | |> Response.encode 13 | 14 | bench "request" do 15 | @request 16 | |> Request.decode 17 | end 18 | 19 | bench "response" do 20 | @response 21 | |> Response.decode 22 | end 23 | 24 | bench "apply request" do 25 | :erlang.apply(Request, :decode, [@request]) 26 | end 27 | 28 | bench "apply response" do 29 | :erlang.apply(Response, :decode, [@response]) 30 | end 31 | 32 | end 33 | -------------------------------------------------------------------------------- /.dialyzer.ignore-warnings: -------------------------------------------------------------------------------- 1 | Unknown function 'Elixir.Protobuf.Serializable.Atom':'__impl__'/1 2 | Unknown function 'Elixir.Protobuf.Serializable.BitString':'__impl__'/1 3 | Unknown function 'Elixir.Protobuf.Serializable.Float':'__impl__'/1 4 | Unknown function 'Elixir.Protobuf.Serializable.Function':'__impl__'/1 5 | Unknown function 'Elixir.Protobuf.Serializable.Integer':'__impl__'/1 6 | Unknown function 'Elixir.Protobuf.Serializable.List':'__impl__'/1 7 | Unknown function 'Elixir.Protobuf.Serializable.Map':'__impl__'/1 8 | Unknown function 'Elixir.Protobuf.Serializable.PID':'__impl__'/1 9 | Unknown function 'Elixir.Protobuf.Serializable.Port':'__impl__'/1 10 | Unknown function 'Elixir.Protobuf.Serializable.Reference':'__impl__'/1 11 | Unknown function 'Elixir.Protobuf.Serializable.Tuple':'__impl__'/1 -------------------------------------------------------------------------------- /test/protobuf/map_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Protobuf.Map.Test do 2 | use Protobuf.Case 3 | 4 | defmodule Msgs do 5 | use Protobuf, from: Path.expand("../proto/map.proto", __DIR__) 6 | end 7 | 8 | @binary <<10, 16, 10, 4, 110, 97, 109, 101, 18, 8, 10, 6, 101, 108, 105, 120, 105, 114>> 9 | 10 | test "can encode map" do 11 | entity = %Msgs.Entity{ 12 | properties: [ 13 | {"name", %Msgs.Value{value: "elixir"}} 14 | ] 15 | } 16 | binary = entity |> Msgs.Entity.encode 17 | assert binary == @binary 18 | end 19 | 20 | test "can decode map" do 21 | entity = @binary |> Msgs.Entity.decode 22 | assert %Msgs.Entity{ 23 | properties: [ 24 | {"name", %Msgs.Value{value: "elixir"}} 25 | ] 26 | } = entity 27 | end 28 | end 29 | 30 | -------------------------------------------------------------------------------- /test/proto/one_of.proto: -------------------------------------------------------------------------------- 1 | message SubMsg { 2 | required string test = 1; 3 | } 4 | 5 | message SampleOneofMsg { 6 | optional string one = 1; 7 | 8 | oneof foo { 9 | string body = 2; 10 | uint32 code = 3; 11 | } 12 | } 13 | 14 | message AdvancedOneofMsg { 15 | optional SubMsg one = 1; 16 | 17 | oneof foo { 18 | SubMsg body = 2; 19 | uint32 code = 3; 20 | } 21 | } 22 | 23 | message ReversedOrderOneOfMsg { 24 | oneof foo { 25 | string body = 1; 26 | uint32 code = 2; 27 | } 28 | 29 | optional string bar = 3; 30 | } 31 | 32 | message SurroundOneOfMsg { 33 | oneof foo { 34 | string body = 1; 35 | uint32 code = 2; 36 | uint32 third = 3; 37 | } 38 | 39 | optional string bar = 4; 40 | 41 | oneof buzz { 42 | string one = 5; 43 | uint32 two = 6; 44 | } 45 | } -------------------------------------------------------------------------------- /.dialyzer.ignore: -------------------------------------------------------------------------------- 1 | :0: Unknown function 'Elixir.Protobuf.Serializable.Atom':'__impl__'/1 2 | :0: Unknown function 'Elixir.Protobuf.Serializable.BitString':'__impl__'/1 3 | :0: Unknown function 'Elixir.Protobuf.Serializable.Float':'__impl__'/1 4 | :0: Unknown function 'Elixir.Protobuf.Serializable.Function':'__impl__'/1 5 | :0: Unknown function 'Elixir.Protobuf.Serializable.Integer':'__impl__'/1 6 | :0: Unknown function 'Elixir.Protobuf.Serializable.List':'__impl__'/1 7 | :0: Unknown function 'Elixir.Protobuf.Serializable.Map':'__impl__'/1 8 | :0: Unknown function 'Elixir.Protobuf.Serializable.PID':'__impl__'/1 9 | :0: Unknown function 'Elixir.Protobuf.Serializable.Port':'__impl__'/1 10 | :0: Unknown function 'Elixir.Protobuf.Serializable.Reference':'__impl__'/1 11 | :0: Unknown function 'Elixir.Protobuf.Serializable.Tuple':'__impl__'/1 12 | -------------------------------------------------------------------------------- /test/proto/nested_one_of.proto: -------------------------------------------------------------------------------- 1 | message Container { 2 | optional string hello = 1; 3 | 4 | oneof msg { 5 | Foo foo = 2; 6 | Bar bar = 3; 7 | } 8 | } 9 | 10 | message Foo { 11 | required bytes foo_id = 1; // conversation id 12 | required uint64 created_at = 2; // creation time of conversation in milliseconds 13 | required FooMetadata metadata = 3; // conversation metadata 14 | } 15 | 16 | message FooMetadata { 17 | oneof type { 18 | GroupFooMetadata group_metadata = 1; // group metadata info 19 | SingleFooMetadata single_metadata = 2; // single conv info 20 | } 21 | } 22 | 23 | message GroupFooMetadata { 24 | repeated string baz_ids = 1; 25 | 26 | optional string avatat = 2; 27 | required string name = 3; 28 | required string foo_owner = 4; 29 | } 30 | 31 | message SingleFooMetadata { 32 | required string baz_id = 1; 33 | } 34 | 35 | message Bar { 36 | required string msg = 1; 37 | } 38 | -------------------------------------------------------------------------------- /lib/exprotobuf/config.ex: -------------------------------------------------------------------------------- 1 | defmodule Protobuf.ConfigError do 2 | defexception [:message] 3 | end 4 | 5 | defmodule Protobuf.Config do 6 | @moduledoc """ 7 | Defines a struct used for configuring the parser behavior. 8 | 9 | ## Options 10 | 11 | * `namespace`: The root module which will define the namespace of generated modules 12 | * `schema`: The schema as a string or a path to a file 13 | * `only`: The list of types to load. If empty, all are loaded. 14 | * `inject`: Flag which when set, determines whether the types loaded are injected into 15 | the current module. If set, then the source proto must only define a single type. 16 | * `use_google_types`: Determines whether or not to include `Google.Protobuf` scalar wrappers, 17 | which can be found in `/priv/google_protobuf.proto` for more details. 18 | 19 | """ 20 | defstruct namespace: nil, 21 | schema: "", 22 | only: [], 23 | inject: false, 24 | from_file: nil, 25 | use_package_names: false, 26 | use_google_types: false, 27 | doc: nil 28 | 29 | def doc_quote(false) do 30 | quote do: @moduledoc(unquote(false)) 31 | end 32 | 33 | def doc_quote(_), do: nil 34 | end 35 | -------------------------------------------------------------------------------- /test/utils/gpb_compile_helper.exs: -------------------------------------------------------------------------------- 1 | defmodule GpbCompileHelper do 2 | 3 | def compile_tmp_proto(msgs) do 4 | compile_tmp_proto(msgs, nil) 5 | end 6 | 7 | def compile_tmp_proto(msgs, func) do 8 | compile_tmp_proto(msgs, [], func) 9 | end 10 | 11 | def compile_tmp_proto(msgs, options, func) do 12 | compile_tmp_proto(msgs, options, find_unused_module(), func) 13 | end 14 | 15 | def compile_tmp_proto(msgs, options, module, func) do 16 | defs = Protobuf.Parser.parse_string!("nofile", msgs, options) 17 | 18 | options = [:binary | options] 19 | 20 | {:ok, ^module, module_binary} = :gpb_compile.msg_defs(module, defs, options) 21 | :code.load_binary(module, '', module_binary) 22 | 23 | if func do 24 | func.(module) 25 | unload(module) 26 | else 27 | module 28 | end 29 | end 30 | 31 | def reload do 32 | Code.unload_files [__ENV__.file] 33 | Code.require_file __ENV__.file 34 | end 35 | 36 | def unload(module) do 37 | :code.purge(module) 38 | :code.delete(module) 39 | end 40 | 41 | def find_unused_module(n \\ 1) do 42 | mod_name_candidate = :'protobuf_test_tmp_#{n}' 43 | case :code.is_loaded(mod_name_candidate) do 44 | false -> mod_name_candidate 45 | {:file, ''} -> find_unused_module(n + 1) 46 | end 47 | end 48 | end 49 | -------------------------------------------------------------------------------- /mix.lock: -------------------------------------------------------------------------------- 1 | %{ 2 | "benchfella": {:hex, :benchfella, "0.3.5", "b2122c234117b3f91ed7b43b6e915e19e1ab216971154acd0a80ce0e9b8c05f5", [:mix], [], "hexpm"}, 3 | "dialyxir": {:hex, :dialyxir, "0.5.1", "b331b091720fd93e878137add264bac4f644e1ddae07a70bf7062c7862c4b952", [:mix], [], "hexpm"}, 4 | "earmark": {:hex, :earmark, "1.3.2", "b840562ea3d67795ffbb5bd88940b1bed0ed9fa32834915125ea7d02e35888a5", [:mix], [], "hexpm"}, 5 | "ex_doc": {:hex, :ex_doc, "0.20.2", "1bd0dfb0304bade58beb77f20f21ee3558cc3c753743ae0ddbb0fd7ba2912331", [:mix], [{:earmark, "~> 1.3", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.10", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm"}, 6 | "gpb": {:hex, :gpb, "4.5.1", "1e575446c9827d092208c433f6cfd9df41a0bcb511d1334cd02d811218362f27", [:make, :rebar], [], "hexpm"}, 7 | "makeup": {:hex, :makeup, "0.8.0", "9cf32aea71c7fe0a4b2e9246c2c4978f9070257e5c9ce6d4a28ec450a839b55f", [:mix], [{:nimble_parsec, "~> 0.5.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm"}, 8 | "makeup_elixir": {:hex, :makeup_elixir, "0.13.0", "be7a477997dcac2e48a9d695ec730b2d22418292675c75aa2d34ba0909dcdeda", [:mix], [{:makeup, "~> 0.8", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm"}, 9 | "nimble_parsec": {:hex, :nimble_parsec, "0.5.0", "90e2eca3d0266e5c53f8fbe0079694740b9c91b6747f2b7e3c5d21966bba8300", [:mix], [], "hexpm"}, 10 | } 11 | -------------------------------------------------------------------------------- /test/utils/gpb_compile_helper_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Protobuf.Test.GpbCompileHelperTest do 2 | use Protobuf.Case 3 | 4 | test "auxiliar compile test function" do 5 | Gpb.compile_tmp_proto ~S[ 6 | message Msg1 { 7 | required uint32 field1 = 1; 8 | } 9 | 10 | message Msg2 { 11 | optional uint32 field1 = 1; 12 | } 13 | 14 | message Msg3 { 15 | enum Type { 16 | TYPE1 = 1; 17 | TYPE2 = 2; 18 | TYPE3 = 3; 19 | } 20 | 21 | message Msg4 { 22 | required uint32 field1 = 1; 23 | } 24 | 25 | required Type field1 = 1; 26 | optional Msg2 field2 = 2; 27 | optional Msg4 field3 = 3; 28 | } 29 | ], fn mod -> 30 | 31 | msgs = [ 32 | [{:Msg1, 10}, <<8, 10>>], 33 | [{:Msg2, :undefined}, <<>>], 34 | [{:Msg3, :TYPE1, :undefined, :undefined}, <<8, 1>>], 35 | [{:Msg3, :TYPE2, {:Msg2, 10}, :undefined}, <<8, 2, 18, 2, 8, 10>>], 36 | [{:Msg3, :TYPE3, {:Msg2, 10}, {:'Msg3.Msg4', 1}}, <<8, 3, 18, 2, 8, 10, 26, 2, 8, 1>>], 37 | ] 38 | 39 | Enum.each(msgs, fn [msg, encoded] -> 40 | msg_name = elem(msg, 0) 41 | assert encoded == mod.encode_msg(msg) 42 | assert msg == mod.decode_msg(mod.encode_msg(msg), msg_name) 43 | end) 44 | end 45 | end 46 | end 47 | -------------------------------------------------------------------------------- /lib/exprotobuf/define_enum.ex: -------------------------------------------------------------------------------- 1 | defmodule Protobuf.DefineEnum do 2 | @moduledoc false 3 | 4 | @doc """ 5 | Defines a new module which contains two functions, atom(value) and value(atom), for 6 | getting either the name or value of an enumeration value. 7 | """ 8 | def def_enum(name, values, inject: inject, doc: doc) do 9 | enum_atoms = Enum.map(values, fn {a, _} -> a end) 10 | enum_values = Enum.map(values, fn {_, v} -> v end) 11 | 12 | contents = 13 | for {atom, value} <- values do 14 | quote do 15 | def value(unquote(atom)), do: unquote(value) 16 | def atom(unquote(value)), do: unquote(atom) 17 | end 18 | end 19 | 20 | contents = 21 | contents ++ 22 | [ 23 | quote do 24 | def values, do: unquote(enum_values) 25 | def atoms, do: unquote(enum_atoms) 26 | end 27 | ] 28 | 29 | if inject do 30 | quote do 31 | unquote(define_typespec(enum_atoms)) 32 | unquote(contents) 33 | def value(_), do: nil 34 | def atom(_), do: nil 35 | end 36 | else 37 | quote do 38 | defmodule unquote(name) do 39 | @moduledoc false 40 | unquote(define_typespec(enum_atoms)) 41 | unquote(Protobuf.Config.doc_quote(doc)) 42 | unquote(contents) 43 | def value(_), do: nil 44 | def atom(_), do: nil 45 | end 46 | end 47 | end 48 | end 49 | 50 | defp define_typespec(enum_atoms) do 51 | quote do 52 | @type t() :: unquote(Protobuf.Utils.define_algebraic_type(enum_atoms)) 53 | end 54 | end 55 | end 56 | -------------------------------------------------------------------------------- /mix.exs: -------------------------------------------------------------------------------- 1 | defmodule Protobuf.Mixfile do 2 | use Mix.Project 3 | 4 | def project do 5 | [app: :exprotobuf, 6 | version: "1.2.17", 7 | elixir: "~> 1.7", 8 | elixirc_paths: elixirc_paths(Mix.env), 9 | preferred_cli_env: [ 10 | bench: :bench, 11 | ], 12 | description: description(), 13 | package: package(), 14 | build_embedded: Mix.env == :prod, 15 | start_permanent: Mix.env == :prod, 16 | consolidate_protocols: Mix.env == :prod, 17 | dialyzer: [ 18 | plt_add_deps: :transitive, 19 | ignore_warnings: ".dialyzer.ignore" 20 | ], 21 | deps: deps()] 22 | end 23 | 24 | def application do 25 | [applications: [:gpb]] 26 | end 27 | 28 | defp description do 29 | """ 30 | exprotobuf provides native encoding/decoding of 31 | protobuf messages via generated modules/structs. 32 | """ 33 | end 34 | 35 | defp package do 36 | [files: ["lib", "mix.exs", "README.md", "LICENSE", "priv"], 37 | maintainers: ["Paul Schoenfelder"], 38 | licenses: ["Apache Version 2.0"], 39 | links: %{"GitHub": "https://github.com/bitwalker/exprotobuf"} ] 40 | end 41 | 42 | defp deps do 43 | [ 44 | {:gpb, "~> 4.0"}, 45 | {:ex_doc, "~> 0.19", only: :dev}, 46 | {:dialyxir, "~> 0.5", only: :dev}, 47 | {:benchfella, "~> 0.3.0", only: [:bench], runtime: false} 48 | ] 49 | end 50 | 51 | # Specifies which paths to compile per environment. 52 | defp elixirc_paths(:test), do: ["lib", "test/support"] 53 | defp elixirc_paths(:bench), do: ["lib", "bench/support"] 54 | defp elixirc_paths(:dev), do: ["lib"] 55 | defp elixirc_paths(_), do: ["lib"] 56 | end 57 | -------------------------------------------------------------------------------- /lib/exprotobuf/delimited.ex: -------------------------------------------------------------------------------- 1 | defmodule Protobuf.Delimited do 2 | @moduledoc """ 3 | Handles serialization/deserialization of multi-message encoded binaries. 4 | """ 5 | 6 | @doc """ 7 | Loops over messages and encodes them. 8 | Also creates a final byte stream which contains the messages delimited by their byte size. 9 | 10 | ## Example 11 | 12 | input = [m1, m2, m3] 13 | output = <> 14 | """ 15 | @spec encode([map]) :: binary 16 | def encode(messages) do 17 | messages 18 | |> Enum.map(&encode_message/1) 19 | |> Enum.join() 20 | end 21 | 22 | @doc """ 23 | Decodes one or more messages in a delimited, encoded binary. 24 | 25 | Input binary should have the following layout: 26 | 27 | <> 28 | 29 | Output will be a list of decoded messages, in the order they appear 30 | in the input binary. If an error occurs, an error tuple will be 31 | returned. 32 | """ 33 | @spec decode(binary, atom) :: [map] | {:error, term} 34 | def decode(bytes, module) do 35 | do_decode(bytes, module, []) 36 | end 37 | 38 | defp do_decode( 39 | <>, 40 | module, 41 | acc 42 | ) do 43 | decoded_message = module.decode(message_bytes) 44 | do_decode(rest, module, [decoded_message | acc]) 45 | end 46 | 47 | defp do_decode(<<>>, _module, acc) do 48 | Enum.reverse(acc) 49 | end 50 | 51 | defp do_decode(rest, _module, _acc) do 52 | {:error, {:delimited_err, {:invalid_binary, rest}}} 53 | end 54 | 55 | defp encode_message(%{__struct__: module} = message) do 56 | encoded_bytes = module.encode(message) 57 | <> 58 | end 59 | end 60 | -------------------------------------------------------------------------------- /test/proto/wrappers.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package Wrappers; 3 | import "Google.Protobuf"; 4 | 5 | message Msg { 6 | 7 | double double_scalar = 1; 8 | float float_scalar = 2; 9 | int64 int64_scalar = 3; 10 | uint64 uint64_scalar = 4; 11 | int32 int32_scalar = 5; 12 | uint32 uint32_scalar = 6; 13 | bool bool_scalar = 7; 14 | string string_scalar = 8; 15 | bytes bytes_scalar = 9; 16 | Os os_scalar = 10; 17 | 18 | Google.Protobuf.DoubleValue double_value = 11; 19 | Google.Protobuf.FloatValue float_value = 12; 20 | Google.Protobuf.Int64Value int64_value = 13; 21 | Google.Protobuf.UInt64Value uint64_value = 14; 22 | Google.Protobuf.Int32Value int32_value = 15; 23 | Google.Protobuf.UInt32Value uint32_value = 16; 24 | Google.Protobuf.BoolValue bool_value = 17; 25 | Google.Protobuf.StringValue string_value = 18; 26 | Google.Protobuf.BytesValue bytes_value = 19; 27 | OsValue os_value = 20; 28 | 29 | oneof oneof_payload { 30 | uint64 uint64_oneof_scalar = 21; 31 | string string_oneof_scalar = 22; 32 | Os os_oneof_scalar = 23; 33 | 34 | Google.Protobuf.UInt64Value uint64_oneof_value = 24; 35 | Google.Protobuf.StringValue string_oneof_value = 25; 36 | OsValue os_oneof_value = 26; 37 | } 38 | 39 | message OsValue { 40 | Os value = 1; 41 | } 42 | 43 | enum Os { 44 | LINUX = 0; 45 | MAC = 1; 46 | WINDOWS = 2; 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /test/protobuf/delimited_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Extprotobuf.Delimited.Test do 2 | use Protobuf.Case 3 | 4 | defmodule Wrapper do 5 | use Protobuf, """ 6 | message User { 7 | required string name = 1; 8 | optional int32 id = 2; 9 | } 10 | """ 11 | end 12 | 13 | @encoded_out <<0, 0, 0, 9, 10, 5, 77, 117, 106, 106, 117, 16, 1>> 14 | @encoded_out_multiple <<0, 0, 0, 9, 10, 5, 77, 117, 106, 106, 117, 16, 1, 0, 0, 0, 9, 10, 5, 77, 117, 106, 106, 117, 16, 1, 0, 0, 0, 9, 10, 5, 77, 117, 106, 106, 117, 16, 1>> 15 | 16 | test "encode creates a valid message for 1 message" do 17 | user = %Wrapper.User{name: "Mujju", id: 1} 18 | encoded_bytes = Protobuf.Delimited.encode([user]) 19 | 20 | encoded_user = Wrapper.User.encode(user) 21 | size = <> 22 | 23 | assert encoded_bytes == size <> encoded_user 24 | assert encoded_bytes == @encoded_out 25 | end 26 | 27 | test "encode creates a valid message for multi message" do 28 | user = %Wrapper.User{name: "Mujju", id: 1} 29 | encoded_bytes = Protobuf.Delimited.encode([user, user, user]) 30 | 31 | encoded_user = Wrapper.User.encode(user) 32 | size = <> 33 | 34 | assert encoded_bytes == String.duplicate(size <> encoded_user, 3) 35 | end 36 | 37 | test "decode creates a valid struct for 1 message" do 38 | assert Protobuf.Delimited.decode(@encoded_out, Wrapper.User) == [%Wrapper.User{name: "Mujju", id: 1}] 39 | end 40 | 41 | test "decode creates a valid struct for 3 message" do 42 | users = Protobuf.Delimited.decode(@encoded_out_multiple, Wrapper.User) 43 | 44 | assert users == Enum.map(1..3, fn(_)-> %Wrapper.User{name: "Mujju", id: 1} end) 45 | end 46 | 47 | test "decode_delimited works" do 48 | assert Wrapper.User.decode_delimited(@encoded_out) == [%Wrapper.User{name: "Mujju", id: 1}] 49 | end 50 | 51 | test "encode_delimited works" do 52 | assert Wrapper.User.encode_delimited([%Wrapper.User{name: "Mujju", id: 1}]) == @encoded_out 53 | end 54 | end 55 | -------------------------------------------------------------------------------- /test/protobuf/from_multiple_files_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Protobuf.FromMultipleFiles.Test do 2 | use Protobuf.Case 3 | 4 | test "generates all messages nested under TopLevel" do 5 | defmodule TopLevel do 6 | use Protobuf, from: [Path.expand("../proto/basic.proto", __DIR__), 7 | Path.expand("../proto/import.proto", __DIR__), 8 | Path.expand("../proto/imported.proto", __DIR__)] 9 | end 10 | 11 | assert %{reason: "hi"} = TopLevel.WrongAuthorizationHttpMessage.new(reason: "hi") 12 | assert %{f1: 255} = TopLevel.Basic.new(f1: 255) 13 | assert %{authorization: "please?"} = TopLevel.WebsocketServerContainer.new(authorization: "please?") 14 | end 15 | 16 | test "prefixs module names with the package names" do 17 | defmodule WithPackageNames do 18 | use Protobuf, from: [Path.expand("../proto/basic.proto", __DIR__), 19 | Path.expand("../proto/import.proto", __DIR__), 20 | Path.expand("../proto/imported.proto", __DIR__)], 21 | use_package_names: true 22 | end 23 | 24 | assert %{reason: "hi"} = WithPackageNames.Authorization.WrongAuthorizationHttpMessage.new(reason: "hi") 25 | assert %{f1: 255} = WithPackageNames.Basic.new(f1: 255) 26 | assert %{authorization: "please?"} = WithPackageNames.Chat.WebsocketServerContainer.new(authorization: "please?") 27 | end 28 | 29 | test "can specify an arbitrary namespace for defining protobuf messages" do 30 | defmodule UnusedNamespace do 31 | use Protobuf, from: [Path.expand("../proto/basic.proto", __DIR__), 32 | Path.expand("../proto/import.proto", __DIR__), 33 | Path.expand("../proto/imported.proto", __DIR__)], 34 | use_package_names: true, 35 | namespace: :"Elixir" 36 | end 37 | 38 | assert %{reason: "hi"} = Authorization.WrongAuthorizationHttpMessage.new(reason: "hi") 39 | assert %{f1: 255} = Basic.new(f1: 255) 40 | assert %{authorization: "please?"} = Chat.WebsocketServerContainer.new(authorization: "please?") 41 | end 42 | end 43 | -------------------------------------------------------------------------------- /imports_upgrade_guide.md: -------------------------------------------------------------------------------- 1 | # Old Behavior 2 | 3 | In previous versions of `exprotobuf` files that had `import "some_other.proto";` statements were automatically handled. 4 | This behavior has been replaced by the ability to load in a list of protobuf files when calling `use Protobuf`. 5 | 6 | ## An Example 7 | 8 | Imagine we had two protobuf files. 9 | 10 | `basic.proto` 11 | 12 | ```protobuf 13 | import "colors.proto"; 14 | 15 | message Basic { 16 | required uint32 id = 1; 17 | optional Color color = 2; 18 | } 19 | ``` 20 | 21 | `colors.proto` 22 | 23 | ```protobuf 24 | enum Color { 25 | WHITE = 0; 26 | BLACK = 1; 27 | GRAY = 2; 28 | RED = 3; 29 | } 30 | ``` 31 | 32 | What we would like to do with these definitions is load them into elixir and do something like: 33 | 34 | ```elixir 35 | Test.Basic.new(id: 123, color: :RED) |> Test.Basic.encode 36 | # => <<8, 123, 16, 3>> 37 | Test.Basic.decode(<<8, 123, 16, 3>>) 38 | # => %Test.Basic{color: :RED, id: 123} 39 | ``` 40 | 41 | ## The Old Behavior 42 | 43 | ```elixir 44 | defmodule Test do 45 | use Protobuf, from: "./test/basic.proto" 46 | end 47 | ``` 48 | 49 | `exprotobuf` would look for the `import "colors.proto";` statement, then try to find that 50 | file, parse it and copy all of its definitions into the same namespace as the Basic message. 51 | This required very little developer effort, but copying definitions had a few drawbacks. 52 | For example, if there were several different files that all used `colors.proto` they would 53 | each have a copy of that definition so there would be multiple elixir modules that all referenced the same enum. 54 | 55 | ## The New Behavior 56 | 57 | ```elixir 58 | defmodule Test do 59 | use Protobuf, from: ["./test/basic.proto","./test/colors.proto"] 60 | end 61 | ``` 62 | 63 | You can now pass a list of proto files to `exprotobuf` and it will parse all of them and resolve all of their names at the same time. 64 | This is a little more work for the developer, but it closely mirrors the way proto files are used in other implementations of protobuf for java and python. 65 | If there were multiple files that all used the `Color` enum, they would all share the same definition and there will be just a single elixir module that represents that enum. 66 | -------------------------------------------------------------------------------- /test/protobuf/nested_one_of_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Protobuf.NestedOneof.Test do 2 | use Protobuf.Case 3 | 4 | defmodule Msgs do 5 | use Protobuf, from: Path.expand("../proto/nested_one_of.proto", __DIR__) 6 | end 7 | 8 | test "can encode nested one_of proto" do 9 | bar = Msgs.Bar.new msg: "msg" 10 | c = Msgs.Container.new hello: "hello", msg: {:bar, bar} 11 | enc_c = Protobuf.Serializable.serialize(c) 12 | 13 | assert is_binary(enc_c) 14 | end 15 | 16 | test "can decode nested one_of proto" do 17 | encoded = <<10, 5, 104, 101, 108, 108, 111, 26, 5, 10, 3, 109, 115, 103>>; 18 | decoded = encoded |> Msgs.Container.decode 19 | 20 | assert %Msgs.Container{} = decoded 21 | end 22 | 23 | test "can encode deeply nested one_of proto" do 24 | sfm = Msgs.SingleFooMetadata.new baz_id: "baz_id" 25 | fm = Msgs.FooMetadata.new type: {:single_metadata, sfm} 26 | foo = Msgs.Foo.new foo_id: "foo_id", created_at: 0, metadata: fm 27 | c = Msgs.Container.new msg: {:foo, foo} 28 | enc_c = Protobuf.Serializable.serialize(c) 29 | 30 | assert is_binary(enc_c) 31 | end 32 | 33 | test "can decode deeply nested one_of proto" do 34 | encoded = <<18, 22, 10, 6, 102, 111, 111, 95, 105, 100, 16, 0, 26, 10, 18, 35 | 8, 10, 6, 98, 97, 122, 95, 105, 100>> 36 | decoded = encoded |> Msgs.Container.decode 37 | 38 | assert %Msgs.Container{} = decoded 39 | end 40 | 41 | test "nested oneof macro in pattern matching" do 42 | alias Msgs.Container 43 | alias Msgs.Foo 44 | alias Msgs.FooMetadata 45 | alias Msgs.SingleFooMetadata 46 | 47 | require Container.OneOf.Msg, as: OneOfMsg 48 | require FooMetadata.OneOf.Type, as: OneOfType 49 | 50 | expected_metadata = %SingleFooMetadata{ 51 | baz_id: "world" 52 | } 53 | 54 | expected_msg = %Foo{ 55 | foo_id: <<1, 2, 3>>, 56 | created_at: 123, 57 | metadata: %FooMetadata{ 58 | type: OneOfType.single_metadata(expected_metadata) 59 | } 60 | } 61 | 62 | %Container{ 63 | hello: "hello", 64 | msg: OneOfMsg.foo(expected_msg) 65 | } 66 | |> Container.encode 67 | |> Container.decode 68 | |> case do 69 | %Container{ 70 | hello: "hello", 71 | msg: OneOfMsg.foo(actual_msg = %Foo{ 72 | foo_id: <<1, 2, 3>>, 73 | created_at: 123, 74 | metadata: %FooMetadata{ 75 | type: OneOfType.single_metadata(actual_metadata) 76 | } 77 | }) 78 | } -> 79 | assert expected_metadata == actual_metadata 80 | assert expected_msg == actual_msg 81 | %Container{} -> 82 | assert false 83 | end 84 | end 85 | 86 | end 87 | -------------------------------------------------------------------------------- /test/protobuf/encoder_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Protobuf.Encoder.Test do 2 | use Protobuf.Case 3 | 4 | defmodule EncoderProto do 5 | use Protobuf, """ 6 | message Msg { 7 | required int32 f1 = 1; 8 | optional int32 f2 = 2; 9 | } 10 | 11 | message WithSubMsg { 12 | required Msg f1 = 1; 13 | } 14 | 15 | message WithRepeatedSubMsg { 16 | repeated Msg f1 = 1; 17 | } 18 | 19 | message extraMsg { 20 | enum msgType { 21 | NACK = 0; 22 | ACK = 1; 23 | } 24 | 25 | required msgType type = 1; 26 | repeated Msg message = 2; 27 | } 28 | 29 | message WithEnum { 30 | enum Version { 31 | V1 = 1; 32 | V2 = 2; 33 | } 34 | 35 | required Version version = 1; 36 | } 37 | 38 | service HelloService { 39 | rpc hello (Msg) returns (Msg); 40 | } 41 | """ 42 | end 43 | 44 | #defmodule ExtensionsProto do 45 | #use Protobuf, """ 46 | #message Msg { 47 | #extensions 200 to max; 48 | #optional string name = 1; 49 | #} 50 | #extend Msg { 51 | #optional string pseudonym = 200; 52 | #} 53 | #""" 54 | #end 55 | 56 | test "fixing nil values to :undefined" do 57 | msg = EncoderProto.Msg.new(f1: 150) 58 | assert <<8, 150, 1>> == Protobuf.Serializable.serialize(msg) 59 | assert <<10, 3, 8, 150, 1>> == Protobuf.Serializable.serialize(EncoderProto.WithSubMsg.new(f1: msg)) 60 | end 61 | 62 | test "fixing a nil value in repeated submsg" do 63 | msg = EncoderProto.WithRepeatedSubMsg.new(f1: [EncoderProto.Msg.new(f1: 1)]) 64 | assert <<10, 2, 8, 1>> == Protobuf.Serializable.serialize(msg) 65 | end 66 | 67 | test "fixing lowercase message and enum references" do 68 | msg = EncoderProto.ExtraMsg.new(type: :ACK, message: [EncoderProto.Msg.new(f1: 1)]) 69 | assert <<8, 1, 18, 2, 8, 1>> == Protobuf.Serializable.serialize(msg) 70 | end 71 | 72 | test "encodes enums" do 73 | msg = EncoderProto.WithEnum.new(version: :'V1') 74 | assert <<8, 1>> == Protobuf.Serializable.serialize(msg) 75 | end 76 | 77 | #test "it can create an extended message" do 78 | #msg = ExtensionsProto.Msg.new(name: "Ron", pseudonym: "Duke Silver") 79 | #assert msg == %ExtensionsProto.Msg{name: "Ron", pseudonym: "Duke Silver"} 80 | #end 81 | 82 | #test "it can encode an extended message" do 83 | #msg = ExtensionsProto.Msg.new(name: "Ron", pseudonym: "Duke Silver") 84 | #assert ExtensionsProto.Msg.encode(msg) == <<10, 3, 82, 111, 110, 194, 12, 11, 68, 117, 107, 101, 32, 83, 105, 108, 118, 101, 114>> 85 | #end 86 | 87 | #test "it can decode an extended message" do 88 | #encoded = <<10, 3, 82, 111, 110, 194, 12, 11, 68, 117, 107, 101, 32, 83, 105, 108, 118, 101, 114>> 89 | #assert ExtensionsProto.Msg.decode(encoded) == %ExtensionsProto.Msg{name: "Ron", pseudonym: "Duke Silver"} 90 | #end 91 | end 92 | -------------------------------------------------------------------------------- /lib/exprotobuf/parser.ex: -------------------------------------------------------------------------------- 1 | defmodule Protobuf.Parser do 2 | defmodule ParserError do 3 | defexception [:message] 4 | end 5 | 6 | def parse_files!(files, options \\ []) do 7 | files 8 | |> Enum.flat_map(fn path -> 9 | schema = File.read!(path) 10 | parse!(path, schema, options) 11 | end) 12 | |> finalize!(options) 13 | end 14 | 15 | def parse_string!(file, string, options \\ []) do 16 | file 17 | |> parse!(string, options) 18 | |> finalize!(options) 19 | end 20 | 21 | defp finalize!(defs, options) do 22 | case :gpb_parse.post_process_all_files(defs, options) do 23 | {:ok, defs} -> 24 | defs 25 | 26 | {:error, error} -> 27 | msg = 28 | case error do 29 | [ref_to_undefined_msg_or_enum: {{root_path, field}, type}] -> 30 | type_ref = 31 | type 32 | |> Enum.map(&Atom.to_string/1) 33 | |> Enum.join() 34 | 35 | invalid_ref = 36 | [field | root_path] 37 | |> Enum.reverse() 38 | |> Enum.map(&Atom.to_string/1) 39 | |> Enum.join() 40 | 41 | "Reference to undefined message or enum #{type_ref} at #{invalid_ref}" 42 | 43 | _ -> 44 | Macro.to_string(error) 45 | end 46 | 47 | raise ParserError, message: msg 48 | end 49 | end 50 | 51 | defp parse(path, string, options) when is_binary(string) or is_list(string) do 52 | case :gpb_scan.string(to_charlist(string)) do 53 | {:ok, tokens, _} -> 54 | lines = 55 | string 56 | |> String.split("\n", parts: :infinity) 57 | |> Enum.count() 58 | 59 | case :gpb_parse.parse(tokens ++ [{:"$end", lines + 1}]) do 60 | {:ok, defs} -> 61 | :gpb_parse.post_process_one_file(to_charlist(path), defs, options) 62 | 63 | error -> 64 | error 65 | end 66 | 67 | error -> 68 | error 69 | end 70 | end 71 | 72 | defp parse!(path, string, options) do 73 | case parse(path, string, options) do 74 | {:ok, defs} -> 75 | defs 76 | 77 | {:error, error} -> 78 | msg = 79 | case error do 80 | [ref_to_undefined_msg_or_enum: {{root_path, field}, type}] -> 81 | type_ref = 82 | type 83 | |> Enum.map(&Atom.to_string/1) 84 | |> Enum.join() 85 | 86 | invalid_ref = 87 | [field | root_path] 88 | |> Enum.reverse() 89 | |> Enum.map(&Atom.to_string/1) 90 | |> Enum.join() 91 | 92 | "Reference to undefined message or enum #{type_ref} at #{invalid_ref}" 93 | 94 | _ when is_binary(error) -> 95 | error 96 | 97 | _ -> 98 | Macro.to_string(error) 99 | end 100 | 101 | raise ParserError, message: msg 102 | end 103 | end 104 | end 105 | -------------------------------------------------------------------------------- /lib/exprotobuf/utils.ex: -------------------------------------------------------------------------------- 1 | defmodule Protobuf.Utils do 2 | @moduledoc false 3 | alias Protobuf.OneOfField 4 | alias Protobuf.Field 5 | 6 | @standard_scalar_wrappers %{ 7 | "Google.Protobuf.DoubleValue" => true, 8 | "Google.Protobuf.FloatValue" => true, 9 | "Google.Protobuf.Int64Value" => true, 10 | "Google.Protobuf.UInt64Value" => true, 11 | "Google.Protobuf.Int32Value" => true, 12 | "Google.Protobuf.UInt32Value" => true, 13 | "Google.Protobuf.BoolValue" => true, 14 | "Google.Protobuf.StringValue" => true, 15 | "Google.Protobuf.BytesValue" => true 16 | } 17 | 18 | defmacro is_scalar(v) do 19 | quote do 20 | (is_atom(unquote(v)) and unquote(v) != nil) or is_number(unquote(v)) or 21 | is_binary(unquote(v)) 22 | end 23 | end 24 | 25 | def is_standard_scalar_wrapper(module) when is_atom(module) do 26 | mod = 27 | module 28 | |> Module.split() 29 | |> Enum.take(-3) 30 | |> Enum.join(".") 31 | 32 | Map.has_key?(@standard_scalar_wrappers, mod) 33 | end 34 | 35 | def is_enum_wrapper(module, enum_module) when is_atom(module) and is_atom(enum_module) do 36 | Atom.to_string(module) == "#{enum_module}Value" 37 | end 38 | 39 | def define_algebraic_type([item]), do: item 40 | 41 | def define_algebraic_type([lhs, rhs]) do 42 | quote do 43 | unquote(lhs) | unquote(rhs) 44 | end 45 | end 46 | 47 | def define_algebraic_type([lhs | rest]) do 48 | quote do 49 | unquote(lhs) | unquote(define_algebraic_type(rest)) 50 | end 51 | end 52 | 53 | def convert_to_record(map, module) do 54 | module.record 55 | |> Enum.reduce([record_name(module)], fn {key, default}, acc -> 56 | value = Map.get(map, key, default) 57 | [value_transform(module, value) | acc] 58 | end) 59 | |> Enum.reverse() 60 | |> List.to_tuple() 61 | end 62 | 63 | def msg_defs(defs) when is_list(defs) do 64 | defs 65 | |> Enum.reduce(%{}, fn 66 | {{:msg, module}, meta}, acc = %{} -> 67 | Map.put(acc, module, do_msg_defs(meta)) 68 | 69 | {{type, _}, _}, acc = %{} when type in [:enum, :extensions, :service, :group] -> 70 | acc 71 | end) 72 | end 73 | 74 | defp do_msg_defs(defs) when is_list(defs) do 75 | defs 76 | |> Enum.reduce(%{}, fn 77 | meta = %Field{name: name}, acc = %{} -> 78 | Map.put(acc, name, meta) 79 | 80 | %OneOfField{name: name, fields: fields}, acc = %{} -> 81 | Map.put(acc, name, do_msg_defs(fields)) 82 | end) 83 | end 84 | 85 | defp record_name(OneOfField), do: :gpb_oneof 86 | defp record_name(Field), do: :field 87 | defp record_name(type), do: type 88 | 89 | defp value_transform(_module, nil), do: :undefined 90 | 91 | defp value_transform(OneOfField, value) when is_list(value) do 92 | Enum.map(value, &convert_to_record(&1, Field)) 93 | end 94 | 95 | defp value_transform(_module, value), do: value 96 | 97 | def convert_from_record(rec, module) do 98 | map = struct(module) 99 | 100 | module.record 101 | |> Enum.with_index() 102 | |> Enum.reduce(map, fn {{key, _default}, idx}, acc -> 103 | # rec has the extra element when defines the record type 104 | value = elem(rec, idx + 1) 105 | Map.put(acc, key, value) 106 | end) 107 | end 108 | end 109 | -------------------------------------------------------------------------------- /lib/exprotobuf/encoder.ex: -------------------------------------------------------------------------------- 1 | defmodule Protobuf.Encoder do 2 | require Protobuf.Utils, as: Utils 3 | alias Protobuf.Field 4 | alias Protobuf.OneOfField 5 | 6 | def encode(%{} = msg, defs) do 7 | fixed_defs = 8 | for {{type, mod}, fields} <- defs, into: [] do 9 | case type do 10 | :msg -> 11 | {{:msg, mod}, 12 | Enum.map(fields, fn field -> 13 | case field do 14 | %OneOfField{} -> 15 | Utils.convert_to_record(field, OneOfField) 16 | 17 | %Field{} -> 18 | Utils.convert_to_record(field, Field) 19 | end 20 | end)} 21 | 22 | type when type in [:enum, :extensions, :service, :group] -> 23 | {{type, mod}, fields} 24 | end 25 | end 26 | 27 | msg 28 | |> wrap_scalars(Utils.msg_defs(defs)) 29 | |> fix_undefined 30 | |> Utils.convert_to_record(msg.__struct__) 31 | |> :gpb.encode_msg(fixed_defs) 32 | end 33 | 34 | defp fix_undefined(%{} = msg) do 35 | Enum.reduce(Map.keys(msg), msg, fn 36 | field, msg -> 37 | original = Map.get(msg, field) 38 | fixed = fix_value(original) 39 | 40 | if original != fixed do 41 | Map.put(msg, field, fixed) 42 | else 43 | msg 44 | end 45 | end) 46 | end 47 | 48 | defp fix_value(nil), do: :undefined 49 | 50 | defp fix_value(values) when is_list(values), 51 | do: Enum.map(values, &fix_value/1) 52 | 53 | defp fix_value(%module{} = value) do 54 | value 55 | |> fix_undefined() 56 | |> Utils.convert_to_record(module) 57 | end 58 | 59 | defp fix_value(value) when is_tuple(value) do 60 | value 61 | |> Tuple.to_list() 62 | |> Enum.map(&fix_value/1) 63 | |> List.to_tuple() 64 | end 65 | 66 | defp fix_value(value), do: value 67 | 68 | defp wrap_scalars(%msg_module{} = msg, %{} = defs) do 69 | msg 70 | |> Map.from_struct() 71 | |> Enum.reduce(msg, fn 72 | # nil is unwrapped 73 | {_, nil}, acc = %_{} -> 74 | acc 75 | 76 | # recursive wrap repeated 77 | {k, v}, acc = %_{} when is_list(v) -> 78 | Map.put(acc, k, Enum.map(v, &wrap_scalars(&1, defs))) 79 | 80 | # recursive wrap message 81 | {k, {oneof, v = %_{}}}, acc = %_{} when is_atom(oneof) -> 82 | Map.put(acc, k, {oneof, wrap_scalars(v, defs)}) 83 | 84 | {k, v = %_{}}, acc = %_{} -> 85 | Map.put(acc, k, wrap_scalars(v, defs)) 86 | 87 | # plain wrap scalar 88 | {k, {oneof, v}}, acc = %_{} when is_atom(oneof) and Utils.is_scalar(v) -> 89 | Map.put(acc, k, {oneof, do_wrap(v, [msg_module, k, oneof], defs)}) 90 | 91 | {k, v}, acc = %_{} when Utils.is_scalar(v) -> 92 | Map.put(acc, k, do_wrap(v, [msg_module, k], defs)) 93 | end) 94 | end 95 | 96 | defp wrap_scalars(v, %{}), do: v 97 | 98 | defp do_wrap(v, keys = [_ | _], defs = %{}) do 99 | case get_in(defs, keys) do 100 | %Field{type: scalar} when is_atom(scalar) -> 101 | v 102 | 103 | %Field{type: {:enum, module}} when is_atom(module) -> 104 | v 105 | 106 | %Field{type: {:msg, module}} when is_atom(module) -> 107 | if Utils.is_standard_scalar_wrapper(module) do 108 | Map.put(module.new, :value, v) 109 | else 110 | do_wrap_enum(v, module, defs) 111 | end 112 | end 113 | end 114 | 115 | defp do_wrap_enum(v, module, defs = %{}) do 116 | case Enum.to_list(Map.get(defs, module)) do 117 | [value: %Field{type: {:enum, enum_module}}] -> 118 | if Utils.is_enum_wrapper(module, enum_module) do 119 | Map.put(module.new, :value, v) 120 | else 121 | v 122 | end 123 | 124 | _ -> 125 | v 126 | end 127 | end 128 | end 129 | -------------------------------------------------------------------------------- /test/protobuf/wrappers_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Protobuf.Wrappers.Test do 2 | use Protobuf.Case 3 | 4 | defmodule Proto do 5 | use Protobuf, 6 | use_package_names: true, 7 | use_google_types: true, 8 | from: Path.expand("../proto/wrappers.proto", __DIR__) 9 | end 10 | 11 | alias Proto.Wrappers.Msg 12 | 13 | setup do 14 | %{ 15 | msg: %Msg{ 16 | double_scalar: 0.0, 17 | float_scalar: 0.0, 18 | int64_scalar: 0, 19 | uint64_scalar: 0, 20 | int32_scalar: 0, 21 | uint32_scalar: 0, 22 | bool_scalar: false, 23 | string_scalar: "", 24 | bytes_scalar: "", 25 | os_scalar: :LINUX, 26 | 27 | double_value: nil, 28 | float_value: nil, 29 | int64_value: nil, 30 | uint64_value: nil, 31 | int32_value: nil, 32 | uint32_value: nil, 33 | bool_value: nil, 34 | string_value: nil, 35 | bytes_value: nil, 36 | os_value: nil, 37 | 38 | oneof_payload: nil 39 | } 40 | } 41 | end 42 | 43 | test "double", %{msg: msg = %Msg{}} do 44 | expected = %Msg{msg | double_scalar: 1.11, double_value: 1.11} 45 | assert expected == expected |> Msg.encode |> Msg.decode 46 | end 47 | 48 | test "float", %{msg: msg = %Msg{}} do 49 | expected = %Msg{msg | float_scalar: 1.0, float_value: 1.0} 50 | assert expected == expected |> Msg.encode |> Msg.decode 51 | end 52 | 53 | test "int64", %{msg: msg = %Msg{}} do 54 | expected = %Msg{msg | int64_scalar: -10, int64_value: -10} 55 | assert expected == expected |> Msg.encode |> Msg.decode 56 | end 57 | 58 | test "uint64", %{msg: msg = %Msg{}} do 59 | expected = %Msg{msg | uint64_scalar: 10, uint64_value: 10} 60 | assert expected == expected |> Msg.encode |> Msg.decode 61 | end 62 | 63 | test "int32", %{msg: msg = %Msg{}} do 64 | expected = %Msg{msg | int32_scalar: -10, int32_value: -10} 65 | assert expected == expected |> Msg.encode |> Msg.decode 66 | end 67 | 68 | test "uint32", %{msg: msg = %Msg{}} do 69 | expected = %Msg{msg | uint32_scalar: 10, uint32_value: 10} 70 | assert expected == expected |> Msg.encode |> Msg.decode 71 | end 72 | 73 | test "bool", %{msg: msg = %Msg{}} do 74 | expected = %Msg{msg | bool_scalar: true, bool_value: true} 75 | assert expected == expected |> Msg.encode |> Msg.decode 76 | end 77 | 78 | test "string", %{msg: msg = %Msg{}} do 79 | expected = %Msg{msg | string_scalar: "hello", string_value: "hello"} 80 | assert expected == expected |> Msg.encode |> Msg.decode 81 | end 82 | 83 | test "bytes", %{msg: msg = %Msg{}} do 84 | expected = %Msg{msg | bytes_scalar: <<224, 224, 224>>, bytes_value: <<224, 224, 224>>} 85 | assert expected == expected |> Msg.encode |> Msg.decode 86 | end 87 | 88 | test "os", %{msg: msg = %Msg{}} do 89 | expected = %Msg{msg | os_scalar: :LINUX, os_value: :LINUX} 90 | assert expected == expected |> Msg.encode |> Msg.decode 91 | end 92 | 93 | test "uint64_oneof_scalar", %{msg: msg = %Msg{}} do 94 | expected = %Msg{msg | oneof_payload: {:uint64_oneof_scalar, 10}} 95 | assert expected == expected |> Msg.encode |> Msg.decode 96 | end 97 | 98 | test "string_oneof_scalar", %{msg: msg = %Msg{}} do 99 | expected = %Msg{msg | oneof_payload: {:string_oneof_scalar, "hello"}} 100 | assert expected == expected |> Msg.encode |> Msg.decode 101 | end 102 | 103 | test "os_oneof_scalar", %{msg: msg = %Msg{}} do 104 | expected = %Msg{msg | oneof_payload: {:os_oneof_scalar, :MAC}} 105 | assert expected == expected |> Msg.encode |> Msg.decode 106 | end 107 | 108 | test "uint64_oneof_value", %{msg: msg = %Msg{}} do 109 | expected = %Msg{msg | oneof_payload: {:uint64_oneof_value, 10}} 110 | assert expected == expected |> Msg.encode |> Msg.decode 111 | end 112 | 113 | test "string_oneof_value", %{msg: msg = %Msg{}} do 114 | expected = %Msg{msg | oneof_payload: {:string_oneof_value, "hello"}} 115 | assert expected == expected |> Msg.encode |> Msg.decode 116 | end 117 | 118 | test "os_oneof_value", %{msg: msg = %Msg{}} do 119 | expected = %Msg{msg | oneof_payload: {:os_oneof_value, :MAC}} 120 | assert expected == expected |> Msg.encode |> Msg.decode 121 | end 122 | 123 | end 124 | -------------------------------------------------------------------------------- /priv/google_protobuf.proto: -------------------------------------------------------------------------------- 1 | // Protocol Buffers - Google's data interchange format 2 | // Copyright 2008 Google Inc. All rights reserved. 3 | // https://developers.google.com/protocol-buffers/ 4 | // 5 | // Redistribution and use in source and binary forms, with or without 6 | // modification, are permitted provided that the following conditions are 7 | // met: 8 | // 9 | // * Redistributions of source code must retain the above copyright 10 | // notice, this list of conditions and the following disclaimer. 11 | // * Redistributions in binary form must reproduce the above 12 | // copyright notice, this list of conditions and the following disclaimer 13 | // in the documentation and/or other materials provided with the 14 | // distribution. 15 | // * Neither the name of Google Inc. nor the names of its 16 | // contributors may be used to endorse or promote products derived from 17 | // this software without specific prior written permission. 18 | // 19 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | // Wrappers for primitive (non-message) types. These types are useful 32 | // for embedding primitives in the `google.protobuf.Any` type and for places 33 | // where we need to distinguish between the absence of a primitive 34 | // typed field and its default value. 35 | // 36 | // These wrappers have no meaningful use within repeated fields as they lack 37 | // the ability to detect presence on individual elements. 38 | // These wrappers have no meaningful use within a map or a oneof since 39 | // individual entries of a map or fields of a oneof can already detect presence. 40 | 41 | syntax = "proto3"; 42 | 43 | package Google.Protobuf; 44 | 45 | option csharp_namespace = "Google.Protobuf.WellKnownTypes"; 46 | option cc_enable_arenas = true; 47 | option go_package = "github.com/golang/protobuf/ptypes/wrappers"; 48 | option java_package = "com.google.protobuf"; 49 | option java_outer_classname = "WrappersProto"; 50 | option java_multiple_files = true; 51 | option objc_class_prefix = "GPB"; 52 | 53 | // Wrapper message for `double`. 54 | // 55 | // The JSON representation for `DoubleValue` is JSON number. 56 | message DoubleValue { 57 | // The double value. 58 | double value = 1; 59 | } 60 | 61 | // Wrapper message for `float`. 62 | // 63 | // The JSON representation for `FloatValue` is JSON number. 64 | message FloatValue { 65 | // The float value. 66 | float value = 1; 67 | } 68 | 69 | // Wrapper message for `int64`. 70 | // 71 | // The JSON representation for `Int64Value` is JSON string. 72 | message Int64Value { 73 | // The int64 value. 74 | int64 value = 1; 75 | } 76 | 77 | // Wrapper message for `uint64`. 78 | // 79 | // The JSON representation for `UInt64Value` is JSON string. 80 | message UInt64Value { 81 | // The uint64 value. 82 | uint64 value = 1; 83 | } 84 | 85 | // Wrapper message for `int32`. 86 | // 87 | // The JSON representation for `Int32Value` is JSON number. 88 | message Int32Value { 89 | // The int32 value. 90 | int32 value = 1; 91 | } 92 | 93 | // Wrapper message for `uint32`. 94 | // 95 | // The JSON representation for `UInt32Value` is JSON number. 96 | message UInt32Value { 97 | // The uint32 value. 98 | uint32 value = 1; 99 | } 100 | 101 | // Wrapper message for `bool`. 102 | // 103 | // The JSON representation for `BoolValue` is JSON `true` and `false`. 104 | message BoolValue { 105 | // The bool value. 106 | bool value = 1; 107 | } 108 | 109 | // Wrapper message for `string`. 110 | // 111 | // The JSON representation for `StringValue` is JSON string. 112 | message StringValue { 113 | // The string value. 114 | string value = 1; 115 | } 116 | 117 | // Wrapper message for `bytes`. 118 | // 119 | // The JSON representation for `BytesValue` is JSON string. 120 | message BytesValue { 121 | // The bytes value. 122 | bytes value = 1; 123 | } 124 | -------------------------------------------------------------------------------- /test/protobuf/decoder_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Protobuf.Decoder.Test do 2 | use Protobuf.Case 3 | alias Protobuf.Decoder, as: D 4 | 5 | test "fix :undefined values to nil in proto2" do 6 | defmodule UndefinedValuesProto2 do 7 | use Protobuf, """ 8 | message Msg { 9 | optional string f1 = 1; 10 | optional int32 f2 = 2; 11 | optional bool f3 = 3; 12 | } 13 | """ 14 | end 15 | 16 | module = UndefinedValuesProto2.Msg 17 | assert %{__struct__: ^module, f1: nil, f2: nil, f3: nil} = D.decode("", module) 18 | end 19 | 20 | test "fix :undefined values to default value in proto3" do 21 | defmodule UndefinedValuesProto3 do 22 | use Protobuf, """ 23 | syntax = "proto3"; 24 | 25 | message Msg { 26 | string f1 = 1; 27 | int32 f2 = 2; 28 | bool f3 = 3; 29 | } 30 | """ 31 | end 32 | 33 | module = UndefinedValuesProto3.Msg 34 | assert %{__struct__: ^module, f1: "", f2: 0, f3: false} = D.decode("", module) 35 | end 36 | 37 | test "fix repeated values" do 38 | defmodule RepeatedValuesProto do 39 | use Protobuf, """ 40 | message Msg { 41 | repeated string f1 = 1; 42 | } 43 | """ 44 | end 45 | 46 | bytes = <<10, 3, 102, 111, 111, 10, 3, 98, 97, 114>> 47 | module = RepeatedValuesProto.Msg 48 | assert %{:__struct__ => ^module, :f1 => ["foo", "bar"]} = D.decode(bytes, RepeatedValuesProto.Msg) 49 | end 50 | 51 | test "fixing string values" do 52 | defmodule FixingStringValuesProto do 53 | use Protobuf, """ 54 | message Msg { 55 | required string f1 = 1; 56 | 57 | message SubMsg { 58 | required string f1 = 1; 59 | } 60 | 61 | optional SubMsg f2 = 2; 62 | } 63 | """ 64 | end 65 | 66 | bytes = <<10,11,?a,?b,?c,0o303,0o245,0o303,0o244,0o303,0o266,0o317,0o276>> 67 | module = FixingStringValuesProto.Msg 68 | submod = FixingStringValuesProto.Msg.SubMsg 69 | assert %{:__struct__ => ^module, :f1 => "abcåäöϾ", :f2 => nil} = D.decode(bytes, FixingStringValuesProto.Msg) 70 | 71 | bytes = <<10, 1, 97, 18, 5, 10, 3, 97, 98, 99>> 72 | assert %{:__struct__ => ^module, :f1 => "a", :f2 => %{:__struct__ => ^submod, :f1 => "abc"}} = D.decode(bytes, FixingStringValuesProto.Msg) 73 | end 74 | 75 | test "enums" do 76 | defmodule EnumsProto do 77 | use Protobuf, """ 78 | message Msg { 79 | message SubMsg { 80 | required uint32 value = 1; 81 | } 82 | 83 | enum Version { 84 | V1 = 1; 85 | V2 = 2; 86 | } 87 | 88 | required Version version = 2; 89 | optional SubMsg sub = 1; 90 | } 91 | """ 92 | end 93 | msg = EnumsProto.Msg.new(version: :'V2') 94 | encoded = EnumsProto.Msg.encode(msg) 95 | decoded = EnumsProto.Msg.decode(encoded) 96 | assert ^msg = decoded 97 | end 98 | 99 | test "extensions" do 100 | defmodule ExtensionsProto do 101 | use Protobuf, """ 102 | message Regular { 103 | required string foo = 1; 104 | } 105 | 106 | message Extended { 107 | required string foo = 1; 108 | extensions 1000 to 1999; 109 | } 110 | 111 | extend Extended { 112 | optional int32 bar = 1003; 113 | } 114 | """ 115 | end 116 | 117 | # for comparison, an extended schema vs an unextended schema. 118 | 119 | # here's the unextended one, note we lose bar: 120 | reg = ExtensionsProto.Regular.new(foo: "hello", bar: 12) 121 | assert reg.foo == "hello" 122 | catch_error(reg.bar) # Regular was not extended. 123 | 124 | reg_encoded = ExtensionsProto.Regular.encode(reg) 125 | reg_decoded = ExtensionsProto.Regular.decode(reg_encoded) 126 | assert ^reg = reg_decoded 127 | 128 | # here's the extended one, note we keep bar even though it's defined outside of the initial definition 129 | ex = ExtensionsProto.Extended.new(foo: "hello", bar: 12) 130 | assert ex.foo == "hello" 131 | assert ex.bar == 12 132 | 133 | ex_encoded = ExtensionsProto.Extended.encode(ex) 134 | ex_decoded = ExtensionsProto.Extended.decode(ex_encoded) 135 | assert ^ex = ex_decoded 136 | 137 | end 138 | 139 | test "complex proto decoding" do 140 | defmodule MumbleProto do 141 | use Protobuf, from: Path.expand("../proto/mumble.proto", __DIR__) 142 | end 143 | 144 | msg = MumbleProto.Authenticate.new(username: "bitwalker") 145 | assert %{username: "bitwalker", password: nil, tokens: []} = msg 146 | end 147 | end 148 | -------------------------------------------------------------------------------- /test/protobuf/one_of_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Protobuf.Oneof.Test do 2 | use Protobuf.Case 3 | 4 | defmodule Msgs do 5 | use Protobuf, from: Path.expand("../proto/one_of.proto", __DIR__) 6 | end 7 | 8 | test "oneof macro in dynamic expression" do 9 | require Msgs.SampleOneofMsg.OneOf.Foo, as: Foo 10 | assert {:body, "HELLO"} == Foo.body("hello" |> String.upcase) 11 | assert {:code, 0} == Foo.code(1 - 1) 12 | end 13 | 14 | test "oneof macro in pattern matching" do 15 | require Msgs.SampleOneofMsg, as: Msg 16 | require Msgs.SampleOneofMsg.OneOf.Foo, as: Foo 17 | %Msg{ 18 | one: "hello", 19 | foo: Foo.body("world" |> String.upcase) 20 | } 21 | |> Msg.encode 22 | |> Msg.decode 23 | |> case do 24 | %Msg{one: "hello", foo: Foo.body("WORLD")} -> assert true 25 | %Msg{} -> assert false 26 | end 27 | end 28 | 29 | test "can create one_of protos" do 30 | msg = Msgs.SampleOneofMsg.new(one: "test", foo: {:body, "xxx"}) 31 | assert %{one: "test", foo: {:body, "xxx"}} = msg 32 | end 33 | 34 | test "can encode simple one_of protos" do 35 | msg = Msgs.SampleOneofMsg.new(one: "test", foo: {:body, "xxx"}) 36 | 37 | encoded = Protobuf.Serializable.serialize(msg) 38 | binary = <<10, 4, 116, 101, 115, 116, 18, 3, 120, 120, 120>> 39 | 40 | assert binary == encoded 41 | end 42 | 43 | test "can decode simple one_of protos" do 44 | binary = <<10, 4, 116, 101, 115, 116, 18, 3, 120, 120, 120>> 45 | 46 | msg = Msgs.SampleOneofMsg.decode(binary) 47 | assert %Msgs.SampleOneofMsg{foo: {:body, "xxx"}, one: "test"} == msg 48 | end 49 | 50 | test "structure parsed simple one_of proto properly" do 51 | defs = Msgs.SampleOneofMsg.defs(:field, :foo) 52 | 53 | assert %Protobuf.OneOfField{fields: [%Protobuf.Field{fnum: 2, name: :body, occurrence: :optional, opts: [], rnum: 3, type: :string}, 54 | %Protobuf.Field{fnum: 3, name: :code, occurrence: :optional, opts: [], rnum: 3, type: :uint32}], name: :foo, rnum: 3} = defs 55 | 56 | end 57 | 58 | test "can create one_of protos with sub messages" do 59 | msg = Msgs.AdvancedOneofMsg.new(one: Msgs.SubMsg.new(test: "xxx"), 60 | foo: {:body, Msgs.SubMsg.new(test: "yyy")}) 61 | 62 | assert %{one: %{test: "xxx"}, foo: {:body, %{test: "yyy"}}} = msg 63 | end 64 | 65 | test "can encode one_of protos with sub messages" do 66 | msg = Msgs.AdvancedOneofMsg.new(one: Msgs.SubMsg.new(test: "xxx"), foo: {:body, Msgs.SubMsg.new(test: "yyy")}) 67 | 68 | 69 | encoded = Protobuf.Serializable.serialize(msg) 70 | 71 | binary = <<10, 5, 10, 3, 120, 120, 120, 18, 5, 10, 3, 121, 121, 121>> 72 | 73 | assert binary == encoded 74 | end 75 | 76 | test "can decode one_of protos with sub messages" do 77 | binary = <<10, 5, 10, 3, 120, 120, 120, 18, 5, 10, 3, 121, 121, 121>> 78 | 79 | msg = Msgs.AdvancedOneofMsg.decode(binary) 80 | assert %Msgs.AdvancedOneofMsg{foo: {:body, Msgs.SubMsg.new(test: "yyy")}, one: Msgs.SubMsg.new(test: "xxx")} == msg 81 | end 82 | 83 | test "can encode one_of protos with one_of field on first position" do 84 | msg = Msgs.ReversedOrderOneOfMsg.new(foo: {:code, 32}, bar: "hi") 85 | enc_msg = Protobuf.Serializable.serialize(msg) 86 | 87 | assert is_binary(enc_msg) 88 | end 89 | 90 | test "can decode one_of protos with one_of field on first position" do 91 | enc_msg= <<16, 32, 26, 2, 104, 105>> 92 | dec_msg = Msgs.ReversedOrderOneOfMsg.decode(enc_msg) 93 | 94 | assert Msgs.ReversedOrderOneOfMsg.new(foo: {:code, 32}, bar: "hi") == dec_msg 95 | end 96 | 97 | test "can encode one_of protos with one_of field on first and third position with three options" do 98 | msg = Msgs.SurroundOneOfMsg.new(foo: {:code, 32}, bar: "hi", buzz: {:one, "3"}) 99 | enc_msg = Protobuf.Serializable.serialize(msg) 100 | 101 | assert is_binary(enc_msg) 102 | end 103 | 104 | test "can decode one_of protos with one_of field on first and third position with three options" do 105 | enc_msg= <<16, 32, 34, 2, 104, 105, 42, 1, 51>> 106 | dec_msg = Msgs.SurroundOneOfMsg.decode(enc_msg) 107 | 108 | assert Msgs.SurroundOneOfMsg.new(foo: {:code, 32}, bar: "hi", buzz: {:one, "3"}) == dec_msg 109 | end 110 | 111 | test "structure parsed with surrounding one_of proto field with three options properly" do 112 | foo_defs = Msgs.SurroundOneOfMsg.defs(:field, :foo) 113 | 114 | assert %Protobuf.OneOfField{fields: [%Protobuf.Field{fnum: 1, name: :body, occurrence: :optional, opts: [], rnum: 2, type: :string}, 115 | %Protobuf.Field{fnum: 2, name: :code, occurrence: :optional, opts: [], rnum: 2, type: :uint32}, 116 | %Protobuf.Field{fnum: 3, name: :third, occurrence: :optional, opts: [], rnum: 2, type: :uint32}], name: :foo, rnum: 2} 117 | = foo_defs 118 | 119 | bar_defs = Msgs.SurroundOneOfMsg.defs(:field, :bar) 120 | 121 | assert %Protobuf.Field{fnum: 4, name: :bar, occurrence: :optional, opts: [], rnum: 3, type: :string} = bar_defs 122 | 123 | buzz_defs = Msgs.SurroundOneOfMsg.defs(:field, :buzz) 124 | 125 | assert %Protobuf.OneOfField{fields: [%Protobuf.Field{fnum: 5, name: :one, occurrence: :optional, opts: [], rnum: 4, type: :string}, 126 | %Protobuf.Field{fnum: 6, name: :two, occurrence: :optional, opts: [], rnum: 4, type: :uint32}], name: :buzz, rnum: 4} 127 | = buzz_defs 128 | end 129 | end 130 | -------------------------------------------------------------------------------- /lib/exprotobuf/decoder.ex: -------------------------------------------------------------------------------- 1 | defmodule Protobuf.Decoder do 2 | use Bitwise, only_operators: true 3 | require Protobuf.Utils, as: Utils 4 | alias Protobuf.Field 5 | alias Protobuf.OneOfField 6 | 7 | # Decode with record/module 8 | def decode(bytes, module) do 9 | defs = 10 | for {{type, mod}, fields} <- module.defs, into: [] do 11 | case type do 12 | :msg -> 13 | {{:msg, mod}, 14 | Enum.map(fields, fn field -> 15 | case field do 16 | %Field{} -> 17 | Utils.convert_to_record(field, Field) 18 | 19 | %OneOfField{} -> 20 | Utils.convert_to_record(field, OneOfField) 21 | end 22 | end)} 23 | 24 | type when type in [:enum, :extensions, :service, :group] -> 25 | {{type, mod}, fields} 26 | end 27 | end 28 | 29 | bytes 30 | |> :gpb.decode_msg(module, defs) 31 | |> Utils.convert_from_record(module) 32 | |> convert_fields() 33 | |> unwrap_scalars(Utils.msg_defs(module.defs)) 34 | end 35 | 36 | def varint(bytes) do 37 | :gpb.decode_varint(bytes) 38 | end 39 | 40 | defp convert_fields(%module{} = msg) do 41 | msg 42 | |> Map.from_struct() 43 | |> Map.keys() 44 | |> Enum.reduce(msg, fn field, msg -> 45 | value = Map.get(msg, field) 46 | 47 | if value == :undefined do 48 | Map.put(msg, field, get_default(module.syntax(), field, module)) 49 | else 50 | convert_field(value, msg, module.defs(:field, field)) 51 | end 52 | end) 53 | end 54 | 55 | defp get_default(:proto2, field, module) do 56 | Map.get(struct(module), field) 57 | end 58 | 59 | defp get_default(:proto3, field, module) do 60 | case module.defs(:field, field) do 61 | %OneOfField{} -> 62 | nil 63 | 64 | x -> 65 | case x.type do 66 | :string -> 67 | "" 68 | 69 | ty -> 70 | case :gpb.proto3_type_default(ty, module.defs) do 71 | :undefined -> 72 | nil 73 | 74 | default -> 75 | default 76 | end 77 | end 78 | end 79 | end 80 | 81 | defp convert_field(value, msg, %Field{name: field, type: type, occurrence: occurrence}) do 82 | case {occurrence, type} do 83 | {:repeated, _} -> 84 | value = 85 | for v <- value do 86 | convert_value(type, v) 87 | end 88 | Map.put(msg, field, value) 89 | 90 | {_, :string} -> 91 | Map.put(msg, field, convert_value(type, value)) 92 | 93 | {_, {:msg, _}} -> 94 | Map.put(msg, field, convert_value(type, value)) 95 | 96 | _ -> 97 | msg 98 | end 99 | end 100 | 101 | defp convert_field({key, inner_value} = value, msg, %OneOfField{name: field}) do 102 | cond do 103 | is_tuple(inner_value) -> 104 | module = elem(inner_value, 0) 105 | 106 | converted_value = 107 | inner_value 108 | |> Utils.convert_from_record(module) 109 | |> convert_fields() 110 | 111 | Map.put(msg, field, {key, converted_value}) 112 | 113 | is_list(inner_value) -> 114 | Map.put(msg, field, {key, convert_value(:string, inner_value)}) 115 | 116 | true -> 117 | Map.put(msg, field, value) 118 | end 119 | end 120 | 121 | defp convert_value(:string, value), 122 | do: :unicode.characters_to_binary(value) 123 | 124 | defp convert_value({:msg, _}, value) do 125 | value 126 | |> Utils.convert_from_record(elem(value, 0)) 127 | |> convert_fields() 128 | end 129 | 130 | defp convert_value({:map, key_type, value_type}, {key, value}) do 131 | {convert_value(key_type, key), convert_value(value_type, value)} 132 | end 133 | 134 | defp convert_value(_, value), 135 | do: value 136 | 137 | defp unwrap_scalars(%msg_module{} = msg, %{} = defs) do 138 | msg 139 | |> Map.from_struct() 140 | |> Enum.reduce(msg, fn 141 | # nil is unwrapped 142 | {_, nil}, %_{} = acc -> 143 | acc 144 | 145 | # recursive unwrap repeated 146 | {k, v}, %_{} = acc when is_list(v) -> 147 | Map.put(acc, k, Enum.map(v, &unwrap_scalars(&1, defs))) 148 | 149 | # unwrap messages 150 | {k, {oneof, %_{} = v}}, %_{} = acc when is_atom(oneof) -> 151 | Map.put(acc, k, {oneof, do_unwrap(v, [msg_module, k, oneof], defs)}) 152 | 153 | {k, %_{} = v}, %_{} = acc -> 154 | Map.put(acc, k, do_unwrap(v, [msg_module, k], defs)) 155 | 156 | # scalars are unwrapped 157 | {_, {oneof, v}}, %_{} = acc when is_atom(oneof) and Utils.is_scalar(v) -> 158 | acc 159 | 160 | {_, v}, %_{} = acc when Utils.is_scalar(v) -> 161 | acc 162 | end) 163 | end 164 | 165 | defp unwrap_scalars(v, %{}), do: v 166 | 167 | defp do_unwrap(v = %_{}, keys = [_ | _], defs = %{}) do 168 | %Field{type: {:msg, module}} = get_in(defs, keys) 169 | 170 | if Utils.is_standard_scalar_wrapper(module) do 171 | v.value 172 | else 173 | do_unwrap_enum(v, module, defs) 174 | end 175 | end 176 | 177 | defp do_unwrap_enum(v = %_{}, module, defs = %{}) do 178 | case Enum.to_list(Map.get(defs, module)) do 179 | [value: %Field{type: {:enum, enum_module}}] -> 180 | if Utils.is_enum_wrapper(module, enum_module) do 181 | v.value 182 | else 183 | # recursive unwrap nested messages 184 | unwrap_scalars(v, defs) 185 | end 186 | 187 | _ -> 188 | # recursive unwrap nested messages 189 | unwrap_scalars(v, defs) 190 | end 191 | end 192 | end 193 | -------------------------------------------------------------------------------- /lib/exprotobuf/builder.ex: -------------------------------------------------------------------------------- 1 | defmodule Protobuf.Builder do 2 | @moduledoc false 3 | 4 | alias Protobuf.Config 5 | 6 | import Protobuf.DefineEnum, only: [def_enum: 3] 7 | import Protobuf.DefineMessage, only: [def_message: 3] 8 | 9 | def define(msgs, %Config{inject: inject} = config) do 10 | # When injecting, use_in is not available, so we don't need to use @before_compile 11 | if inject do 12 | quote location: :keep do 13 | Module.register_attribute(__MODULE__, :use_in, accumulate: true) 14 | import unquote(__MODULE__), only: [use_in: 2] 15 | 16 | unless is_nil(unquote(config.from_file)) do 17 | case unquote(config.from_file) do 18 | file when is_binary(file) -> 19 | @external_resource file 20 | 21 | files when is_list(files) -> 22 | for file <- files do 23 | @external_resource file 24 | end 25 | end 26 | end 27 | 28 | @config unquote(Macro.escape(Map.to_list(%{config | :schema => nil}))) 29 | @msgs unquote(Macro.escape(msgs)) 30 | contents = unquote(__MODULE__).generate(@msgs, @config) 31 | Module.eval_quoted(__MODULE__, contents, [], __ENV__) 32 | end 33 | else 34 | quote do 35 | Module.register_attribute(__MODULE__, :use_in, accumulate: true) 36 | import unquote(__MODULE__), only: [use_in: 2] 37 | 38 | unless is_nil(unquote(config.from_file)) do 39 | case unquote(config.from_file) do 40 | file when is_binary(file) -> 41 | @external_resource file 42 | 43 | files when is_list(files) -> 44 | for file <- files do 45 | @external_resource file 46 | end 47 | end 48 | end 49 | 50 | @config unquote(Macro.escape(Map.to_list(%{config | schema: nil}))) 51 | @msgs unquote(Macro.escape(msgs)) 52 | @before_compile unquote(__MODULE__) 53 | end 54 | end 55 | end 56 | 57 | defmacro __before_compile__(_env) do 58 | quote location: :keep do 59 | contents = unquote(__MODULE__).generate(@msgs, @config) 60 | Module.eval_quoted(__MODULE__, contents, [], __ENV__) 61 | end 62 | end 63 | 64 | # Cache use instructions 65 | defmacro use_in(module, use_module) do 66 | module = :"#{__CALLER__.module}.#{module}" 67 | use_module = quote do: use(unquote(use_module)) 68 | 69 | quote location: :keep do 70 | @use_in {unquote(module), unquote(Macro.escape(use_module))} 71 | end 72 | end 73 | 74 | # Generate code of records (message and enum) 75 | def generate(msgs, config) do 76 | only = Keyword.get(config, :only, []) 77 | inject = Keyword.get(config, :inject, false) 78 | doc = Keyword.get(config, :doc, true) 79 | ns = Keyword.get(config, :namespace) 80 | 81 | quotes = 82 | for {{item_type, item_name}, fields} <- msgs, 83 | item_type in [:msg, :proto3_msg, :enum], 84 | into: [] do 85 | if only != [] do 86 | is_child? = 87 | Enum.any?(only, fn o -> 88 | o != item_name and is_child_type?(item_name, o) 89 | end) 90 | 91 | last_mod = last_module(item_name) 92 | if last_mod in only or is_child? do 93 | case item_type do 94 | :msg when is_child? -> 95 | item_name = fix_ns(item_name, ns) 96 | 97 | def_message(item_name, fields, 98 | inject: false, 99 | doc: doc, 100 | syntax: :proto2 101 | ) 102 | 103 | :msg -> 104 | def_message(ns, fields, inject: inject, doc: doc, syntax: :proto2) 105 | 106 | :proto3_msg when is_child? -> 107 | item_name = fix_ns(item_name, ns) 108 | 109 | def_message(item_name, fields, 110 | inject: false, 111 | doc: doc, 112 | syntax: :proto3 113 | ) 114 | 115 | :proto3_msg -> 116 | def_message(ns, fields, inject: inject, doc: doc, syntax: :proto3) 117 | 118 | :enum when is_child? -> 119 | item_name = fix_ns(item_name, ns) 120 | def_enum(item_name, fields, inject: false, doc: doc) 121 | 122 | :enum -> 123 | def_enum(ns, fields, inject: inject, doc: doc) 124 | 125 | _ -> 126 | [] 127 | end 128 | end 129 | else 130 | case item_type do 131 | :msg -> 132 | def_message(item_name, fields, inject: false, doc: doc, syntax: :proto2) 133 | 134 | :proto3_msg -> 135 | def_message(item_name, fields, inject: false, doc: doc, syntax: :proto3) 136 | 137 | :enum -> 138 | def_enum(item_name, fields, inject: false, doc: doc) 139 | 140 | _ -> 141 | [] 142 | end 143 | end 144 | end 145 | 146 | unified_msgs = Enum.map(msgs, &unify_msg_types/1) 147 | 148 | # Global defs helper 149 | quotes ++ 150 | [ 151 | quote do 152 | def defs do 153 | unquote(Macro.escape(unified_msgs, unquote: true)) 154 | end 155 | end 156 | ] 157 | end 158 | 159 | defp unify_msg_types({{:proto3_msg, name}, fields}), do: {{:msg, name}, fields} 160 | defp unify_msg_types(other), do: other 161 | 162 | defp is_child_type?(child, type) do 163 | [parent | _] = 164 | child 165 | |> Atom.to_string() 166 | |> String.split(".", parts: :infinity) 167 | 168 | Atom.to_string(type) == parent 169 | end 170 | 171 | defp fix_ns(name, ns) do 172 | name_parts = 173 | name 174 | |> Atom.to_string() 175 | |> String.split(".", parts: :infinity) 176 | 177 | ns_parts = 178 | ns 179 | |> Atom.to_string() 180 | |> String.split(".", parts: :infinity) 181 | 182 | name_parts = name_parts -- ns_parts 183 | 184 | module = 185 | name_parts 186 | |> Enum.join() 187 | |> String.to_atom() 188 | 189 | :"#{ns}.#{module}" 190 | end 191 | 192 | defp last_module(namespace) do 193 | namespace |> Module.split() |> List.last() |> String.to_atom() 194 | end 195 | end 196 | -------------------------------------------------------------------------------- /test/proto/mumble.proto: -------------------------------------------------------------------------------- 1 | package MumbleProto; 2 | 3 | option optimize_for = SPEED; 4 | 5 | message Version { 6 | optional uint32 version = 1; 7 | optional string release = 2; 8 | optional string os = 3; 9 | optional string os_version = 4; 10 | } 11 | 12 | message UDPTunnel { 13 | required bytes packet = 1; 14 | } 15 | 16 | message Authenticate { 17 | optional string username = 1; 18 | optional string password = 2; 19 | repeated string tokens = 3; 20 | repeated int32 celt_versions = 4; 21 | optional bool opus = 5 [default = false]; 22 | } 23 | 24 | message Ping { 25 | optional uint64 timestamp = 1; 26 | optional uint32 good = 2; 27 | optional uint32 late = 3; 28 | optional uint32 lost = 4; 29 | optional uint32 resync = 5; 30 | optional uint32 udp_packets = 6; 31 | optional uint32 tcp_packets = 7; 32 | optional float udp_ping_avg = 8; 33 | optional float udp_ping_var = 9; 34 | optional float tcp_ping_avg = 10; 35 | optional float tcp_ping_var = 11; 36 | } 37 | 38 | message Reject { 39 | enum RejectType { 40 | None = 0; 41 | WrongVersion = 1; 42 | InvalidUsername = 2; 43 | WrongUserPW = 3; 44 | WrongServerPW = 4; 45 | UsernameInUse = 5; 46 | ServerFull = 6; 47 | NoCertificate = 7; 48 | AuthenticatorFail = 8; 49 | } 50 | optional RejectType type = 1; 51 | optional string reason = 2; 52 | } 53 | 54 | message ServerSync { 55 | optional uint32 session = 1; 56 | optional uint32 max_bandwidth = 2; 57 | optional string welcome_text = 3; 58 | optional uint64 permissions = 4; 59 | } 60 | 61 | message ChannelRemove { 62 | required uint32 channel_id = 1; 63 | } 64 | 65 | message ChannelState { 66 | optional uint32 channel_id = 1; 67 | optional uint32 parent = 2; 68 | optional string name = 3; 69 | repeated uint32 links = 4; 70 | optional string description = 5; 71 | repeated uint32 links_add = 6; 72 | repeated uint32 links_remove = 7; 73 | optional bool temporary = 8 [default = false]; 74 | optional int32 position = 9 [default = 0]; 75 | optional bytes description_hash = 10; 76 | } 77 | 78 | message UserRemove { 79 | required uint32 session = 1; 80 | optional uint32 actor = 2; 81 | optional string reason = 3; 82 | optional bool ban = 4; 83 | } 84 | 85 | message UserState { 86 | optional uint32 session = 1; 87 | optional uint32 actor = 2; 88 | optional string name = 3; 89 | optional uint32 user_id = 4; 90 | optional uint32 channel_id = 5; 91 | optional bool mute = 6; 92 | optional bool deaf = 7; 93 | optional bool suppress = 8; 94 | optional bool self_mute = 9; 95 | optional bool self_deaf = 10; 96 | optional bytes texture = 11; 97 | optional bytes plugin_context = 12; 98 | optional string plugin_identity = 13; 99 | optional string comment = 14; 100 | optional string hash = 15; 101 | optional bytes comment_hash = 16; 102 | optional bytes texture_hash = 17; 103 | optional bool priority_speaker = 18; 104 | optional bool recording = 19; 105 | } 106 | 107 | message BanList { 108 | message BanEntry { 109 | required bytes address = 1; 110 | required uint32 mask = 2; 111 | optional string name = 3; 112 | optional string hash = 4; 113 | optional string reason = 5; 114 | optional string start = 6; 115 | optional uint32 duration = 7; 116 | } 117 | repeated BanEntry bans = 1; 118 | optional bool query = 2 [default = false]; 119 | } 120 | 121 | message TextMessage { 122 | optional uint32 actor = 1; 123 | repeated uint32 session = 2; 124 | repeated uint32 channel_id = 3; 125 | repeated uint32 tree_id = 4; 126 | required string message = 5; 127 | } 128 | 129 | message PermissionDenied { 130 | enum DenyType { 131 | Text = 0; 132 | Permission = 1; 133 | SuperUser = 2; 134 | ChannelName = 3; 135 | TextTooLong = 4; 136 | H9K = 5; 137 | TemporaryChannel = 6; 138 | MissingCertificate = 7; 139 | UserName = 8; 140 | ChannelFull = 9; 141 | NestingLimit = 10; 142 | } 143 | optional uint32 permission = 1; 144 | optional uint32 channel_id = 2; 145 | optional uint32 session = 3; 146 | optional string reason = 4; 147 | optional DenyType type = 5; 148 | optional string name = 6; 149 | } 150 | 151 | message ACL { 152 | message ChanGroup { 153 | required string name = 1; 154 | optional bool inherited = 2 [default = true]; 155 | optional bool inherit = 3 [default = true]; 156 | optional bool inheritable = 4 [default = true]; 157 | repeated uint32 add = 5; 158 | repeated uint32 remove = 6; 159 | repeated uint32 inherited_members = 7; 160 | } 161 | message ChanACL { 162 | optional bool apply_here = 1 [default = true]; 163 | optional bool apply_subs = 2 [default = true]; 164 | optional bool inherited = 3 [default = true]; 165 | optional uint32 user_id = 4; 166 | optional string group = 5; 167 | optional uint32 grant = 6; 168 | optional uint32 deny = 7; 169 | } 170 | required uint32 channel_id = 1; 171 | optional bool inherit_acls = 2 [default = true]; 172 | repeated ChanGroup groups = 3; 173 | repeated ChanACL acls = 4; 174 | optional bool query = 5 [default = false]; 175 | } 176 | 177 | message QueryUsers { 178 | repeated uint32 ids = 1; 179 | repeated string names = 2; 180 | } 181 | 182 | message CryptSetup { 183 | optional bytes key = 1; 184 | optional bytes client_nonce = 2; 185 | optional bytes server_nonce = 3; 186 | } 187 | 188 | message ContextActionModify { 189 | enum Context { 190 | Server = 0x01; 191 | Channel = 0x02; 192 | User = 0x04; 193 | } 194 | enum Operation { 195 | Add = 0; 196 | Remove = 1; 197 | } 198 | required string action = 1; 199 | optional string text = 2; 200 | optional uint32 context = 3; 201 | optional Operation operation = 4; 202 | } 203 | 204 | message ContextAction { 205 | optional uint32 session = 1; 206 | optional uint32 channel_id = 2; 207 | required string action = 3; 208 | } 209 | 210 | message UserList { 211 | message User { 212 | required uint32 user_id = 1; 213 | optional string name = 2; 214 | optional string last_seen = 3; 215 | optional uint32 last_channel = 4; 216 | } 217 | repeated User users = 1; 218 | } 219 | 220 | message VoiceTarget { 221 | message Target { 222 | repeated uint32 session = 1; 223 | optional uint32 channel_id = 2; 224 | optional string group = 3; 225 | optional bool links = 4 [default = false]; 226 | optional bool children = 5 [default = false]; 227 | } 228 | optional uint32 id = 1; 229 | repeated Target targets = 2; 230 | } 231 | 232 | message PermissionQuery { 233 | optional uint32 channel_id = 1; 234 | optional uint32 permissions = 2; 235 | optional bool flush = 3 [default = false]; 236 | } 237 | 238 | message CodecVersion { 239 | required int32 alpha = 1; 240 | required int32 beta = 2; 241 | required bool prefer_alpha = 3 [default = true]; 242 | optional bool opus = 4 [default = false]; 243 | } 244 | 245 | message UserStats { 246 | message Stats { 247 | optional uint32 good = 1; 248 | optional uint32 late = 2; 249 | optional uint32 lost = 3; 250 | optional uint32 resync = 4; 251 | } 252 | 253 | optional uint32 session = 1; 254 | optional bool stats_only = 2 [default = false]; 255 | repeated bytes certificates = 3; 256 | optional Stats from_client = 4; 257 | optional Stats from_server = 5; 258 | 259 | optional uint32 udp_packets = 6; 260 | optional uint32 tcp_packets = 7; 261 | optional float udp_ping_avg = 8; 262 | optional float udp_ping_var = 9; 263 | optional float tcp_ping_avg = 10; 264 | optional float tcp_ping_var = 11; 265 | 266 | optional Version version = 12; 267 | repeated int32 celt_versions = 13; 268 | optional bytes address = 14; 269 | optional uint32 bandwidth = 15; 270 | optional uint32 onlinesecs = 16; 271 | optional uint32 idlesecs = 17; 272 | optional bool strong_certificate = 18 [default = false]; 273 | optional bool opus = 19 [default = false]; 274 | } 275 | 276 | message RequestBlob { 277 | repeated uint32 session_texture = 1; 278 | repeated uint32 session_comment = 2; 279 | repeated uint32 channel_description = 3; 280 | } 281 | 282 | message ServerConfig { 283 | optional uint32 max_bandwidth = 1; 284 | optional string welcome_text = 2; 285 | optional bool allow_html = 3; 286 | optional uint32 message_length = 4; 287 | optional uint32 image_message_length = 5; 288 | } 289 | 290 | message SuggestConfig { 291 | optional uint32 version = 1; 292 | optional bool positional = 2; 293 | optional bool push_to_talk = 3; 294 | } 295 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Protocol Buffers for Elixir 2 | 3 | exprotobuf works by building module/struct definitions from a [Google Protocol Buffer](https://code.google.com/p/protobuf) 4 | schema. This allows you to work with protocol buffers natively in Elixir, with easy decoding/encoding for transport across the 5 | wire. 6 | 7 | [![Build Status](https://travis-ci.org/bitwalker/exprotobuf.svg?branch=master)](https://travis-ci.org/bitwalker/exprotobuf) 8 | [![Hex.pm Version](http://img.shields.io/hexpm/v/exprotobuf.svg?style=flat)](https://hex.pm/packages/exprotobuf) 9 | 10 | ## Features 11 | 12 | * Load protobuf from file or string 13 | * Respects the namespace of messages 14 | * Allows you to specify which modules should be loaded in the definition of records 15 | * Currently uses [gpb](https://github.com/tomas-abrahamsson/gpb) for protobuf schema parsing 16 | 17 | TODO: 18 | 19 | * Clean up code/tests 20 | 21 | ## Breaking Changes 22 | 23 | The 1.0 release removed the feature of handling `import "...";` statements. 24 | Please see [the imports upgrade guide](imports_upgrade_guide.md) for details if you were using this feature. 25 | 26 | ## Getting Started 27 | 28 | Add exprotobuf as a dependency to your project: 29 | 30 | ```elixir 31 | defp deps do 32 | [{:exprotobuf, "~> x.x.x"}] 33 | end 34 | ``` 35 | 36 | Then run `mix deps.get` to fetch. 37 | 38 | Add exprotobuf to applications list: 39 | 40 | ```elixir 41 | def application do 42 | [applications: [:exprotobuf]] 43 | end 44 | ``` 45 | 46 | ## Usage 47 | 48 | Usage of exprotobuf boils down to a single `use` statement within one or 49 | more modules in your project. 50 | 51 | Let's start with the most basic of usages: 52 | 53 | ### Define from a string 54 | 55 | ```elixir 56 | defmodule Messages do 57 | use Protobuf, """ 58 | message Msg { 59 | message SubMsg { 60 | required uint32 value = 1; 61 | } 62 | 63 | enum Version { 64 | V1 = 1; 65 | V2 = 2; 66 | } 67 | 68 | required Version version = 2; 69 | optional SubMsg sub = 1; 70 | } 71 | """ 72 | end 73 | ``` 74 | 75 | ```elixir 76 | iex> msg = Messages.Msg.new(version: :'V2') 77 | %Messages.Msg{version: :V2, sub: nil} 78 | iex> encoded = Messages.Msg.encode(msg) 79 | <<16, 2>> 80 | iex> Messages.Msg.decode(encoded) 81 | %Messages.Msg{version: :V2, sub: nil} 82 | ``` 83 | 84 | The above code takes the provided protobuf schema as a string, and 85 | generates modules/structs for the types it defines. In this case, there 86 | would be a Msg module, containing a SubMsg and Version module. The 87 | properties defined for those values are keys in the struct belonging to 88 | each. Enums do not generate structs, but a specialized module with two 89 | functions: `atom(x)` and `value(x)`. These will get either the name of 90 | the enum value, or it's associated value. 91 | 92 | Values defined in the schema using the `oneof` construct are represented with tuples: 93 | 94 | ```elixir 95 | defmodule Messages do 96 | use Protobuf, """ 97 | message Msg { 98 | oneof choice { 99 | string first = 1; 100 | int32 second = 2; 101 | } 102 | } 103 | """ 104 | end 105 | ``` 106 | 107 | ```elixir 108 | iex> msg = Messages.Msg.new(choice: {:second, 42}) 109 | %Messages.Msg{choice: {:second, 42}} 110 | iex> encoded = Messages.Msg.encode(msg) 111 | <<16, 42>> 112 | ``` 113 | 114 | ### Define from a file 115 | 116 | ```elixir 117 | defmodule Messages do 118 | use Protobuf, from: Path.expand("../proto/messages.proto", __DIR__) 119 | end 120 | ``` 121 | 122 | This is equivalent to the above, if you assume that `messages.proto` 123 | contains the same schema as in the string of the first example. 124 | 125 | ### Loading all definitions from a set of files 126 | 127 | ```elixir 128 | defmodule Protobufs do 129 | use Protobuf, from: Path.wildcard(Path.expand("../definitions/**/*.proto", __DIR__)) 130 | end 131 | ``` 132 | 133 | ```elixir 134 | iex> Protobufs.Msg.new(v: :V1) 135 | %Protobufs.Msg{v: :V1} 136 | iex> %Protobufs.OtherMessage{middle_name: "Danger"} 137 | %Protobufs.OtherMessage{middle_name: "Danger"} 138 | ``` 139 | 140 | This will load all the various definitions in your `.proto` files and 141 | allow them to share definitions like enums or messages between them. 142 | 143 | ### Customizing Generated Module Names 144 | 145 | In some cases your library of protobuf definitions might already contain some 146 | namespaces that you would like to keep. 147 | In this case you will probably want to pass the `use_package_names: true` option. 148 | Let's say you had a file called `protobufs/example.proto` that contained: 149 | 150 | ```protobuf 151 | package world; 152 | message Example { 153 | enum Continent { 154 | ANTARCTICA = 0; 155 | EUROPE = 1; 156 | } 157 | 158 | optional Continent continent = 1; 159 | optional uint32 id = 2; 160 | } 161 | ``` 162 | 163 | You could load that file (and everything else in the protobufs directory) by doing: 164 | 165 | ```elixir 166 | defmodule Definitions do 167 | use Protobuf, from: Path.wildcard("protobufs/*.proto"), use_package_names: true 168 | end 169 | ``` 170 | 171 | ```elixir 172 | iex> Definitions.World.Example.new(continent: :EUROPE) 173 | %Definitions.World.Example{continent: :EUROPE} 174 | ``` 175 | 176 | You might also want to define all of these modules in the top-level namespace. You 177 | can do this by passing an explicit `namespace: :"Elixir"` option. 178 | 179 | ```elixir 180 | defmodule Definitions do 181 | use Protobuf, from: Path.wildcard("protobufs/*.proto"), 182 | use_package_names: true, 183 | namespace: :"Elixir" 184 | end 185 | ``` 186 | 187 | ```elixir 188 | iex> World.Example.new(continent: :EUROPE) 189 | %World.Example{continent: :EUROPE} 190 | ``` 191 | 192 | Now you can use just the package names and message names that your team is already 193 | familiar with. 194 | 195 | ### Inject a definition into an existing module 196 | 197 | This is useful when you only have a single type, or if you want to pull 198 | the module definition into the current module instead of generating a 199 | new one. 200 | 201 | ```elixir 202 | defmodule Msg do 203 | use Protobuf, from: Path.expand("../proto/messages.proto", __DIR__), inject: true 204 | 205 | def update(msg, key, value), do: Map.put(msg, key, value) 206 | end 207 | ``` 208 | 209 | ```elixir 210 | iex> %Msg{} 211 | %Msg{v: :V1} 212 | iex> Msg.update(%Msg{}, :v, :V2) 213 | %Msg{v: :V2} 214 | ``` 215 | 216 | As you can see, Msg is no longer created as a nested module, but is 217 | injected right at the top level. I find this approach to be a lot 218 | cleaner than `use_in`, but may not work in all use cases. 219 | 220 | ### Inject a specific type from a larger subset of types 221 | 222 | When you have a large schema, but perhaps only care about a small subset 223 | of those types, you can use `:only`: 224 | 225 | ```elixir 226 | defmodule Messages do 227 | use Protobuf, from: Path.expand("../proto/messages.proto", __DIR__), 228 | only: [:TypeA, :TypeB] 229 | end 230 | ``` 231 | 232 | Assuming that the provided .proto file contains multiple type 233 | definitions, the above code would extract only TypeA and TypeB as nested 234 | modules. Keep in mind your dependencies, if you select a child type 235 | which depends on a parent, or another top-level type, exprotobuf may 236 | fail, or your code may fail at runtime. 237 | 238 | You may only combine `:only` with `:inject` when `:only` is a single 239 | type, or a list containing a single type. This is due to the restriction 240 | of one struct per module. Theoretically you should be able to pass `:only` 241 | with multiple types, as long all but one of the types is an enum, since 242 | enums are just generated as modules, this does not currently work 243 | though. 244 | 245 | ### Extend generated modules via `use_in` 246 | 247 | If you need to add behavior to one of the generated modules, `use_in` 248 | will help you. The tricky part is that the struct for the module you 249 | `use_in` will not be defined yet, so you can't rely on it in your 250 | functions. You can still work with the structs via the normal Maps API, 251 | but you lose compile-time guarantees. I would recommend favoring 252 | `:inject` over this when possible, as it's a much cleaner solution. 253 | 254 | ```elixir 255 | defmodule Messages do 256 | use Protobuf, " 257 | message Msg { 258 | enum Version { 259 | V1 = 1; 260 | V2 = 1; 261 | } 262 | required Version v = 1; 263 | } 264 | " 265 | 266 | defmodule MsgHelpers do 267 | defmacro __using__(_opts) do 268 | quote do 269 | def convert_to_record(msg) do 270 | msg 271 | |> Map.to_list 272 | |> Enum.reduce([], fn {_key, value}, acc -> [value | acc] end) 273 | |> Enum.reverse 274 | |> list_to_tuple 275 | end 276 | end 277 | end 278 | end 279 | 280 | use_in "Msg", MsgHelpers 281 | end 282 | ``` 283 | 284 | ```elixir 285 | iex> Messages.Msg.new |> Messages.Msg.convert_to_record 286 | {Messages.Msg, :V1} 287 | ``` 288 | 289 | ## Attribution/License 290 | 291 | exprotobuf is a fork of the azukiaapp/elixir-protobuf project, both of which are released under Apache 2 License. 292 | 293 | Check LICENSE files for more information. 294 | -------------------------------------------------------------------------------- /lib/exprotobuf.ex: -------------------------------------------------------------------------------- 1 | defmodule Protobuf do 2 | alias Protobuf.Parser 3 | alias Protobuf.Builder 4 | alias Protobuf.Config 5 | alias Protobuf.ConfigError 6 | alias Protobuf.Field 7 | alias Protobuf.OneOfField 8 | alias Protobuf.Utils 9 | 10 | defmacro __using__(schema) when is_binary(schema) do 11 | config = %Config{namespace: __CALLER__.module, schema: schema} 12 | 13 | config 14 | |> parse(__CALLER__) 15 | |> Builder.define(config) 16 | end 17 | 18 | defmacro __using__([schema | opts]) when is_binary(schema) do 19 | namespace = __CALLER__.module 20 | 21 | config = 22 | case Enum.into(opts, %{}) do 23 | %{only: only, inject: true} -> 24 | types = parse_only(only, __CALLER__) 25 | 26 | case types do 27 | [] -> 28 | raise ConfigError, 29 | error: "You must specify a type using :only when combined with inject: true" 30 | 31 | [_type] -> 32 | %Config{namespace: namespace, schema: schema, only: types, inject: true} 33 | end 34 | 35 | %{only: only} -> 36 | %Config{namespace: namespace, schema: schema, only: parse_only(only, __CALLER__)} 37 | 38 | %{inject: true} -> 39 | only = last_module(namespace) 40 | %Config{namespace: namespace, schema: schema, only: [only], inject: true} 41 | end 42 | 43 | use_google_types = get_in(opts, [:use_google_types]) || false 44 | 45 | %Config{config | use_google_types: use_google_types} 46 | |> parse(__CALLER__) 47 | |> Builder.define(config) 48 | end 49 | 50 | defmacro __using__(opts) when is_list(opts) do 51 | {namespace, opts} = Keyword.pop(opts, :namespace, __CALLER__.module) 52 | {doc, opts} = Keyword.pop(opts, :doc, nil) 53 | {use_google_types, opts} = Keyword.pop(opts, :use_google_types, false) 54 | opts = Enum.into(opts, %{}) 55 | 56 | config = 57 | case opts do 58 | %{from: file, use_package_names: use_package_names, only: only, inject: true} -> 59 | types = parse_only(only, __CALLER__) 60 | 61 | case types do 62 | [] -> 63 | raise ConfigError, 64 | error: "You must specify a type using :only when combined with inject: true" 65 | 66 | [_type] -> 67 | %Config{ 68 | namespace: namespace, 69 | only: types, 70 | inject: true, 71 | use_package_names: use_package_names, 72 | from_file: file, 73 | doc: doc 74 | } 75 | end 76 | 77 | only = last_module(namespace) 78 | 79 | %Config{ 80 | namespace: namespace, 81 | only: [only], 82 | from_file: file, 83 | inject: true, 84 | use_package_names: use_package_names, 85 | doc: doc 86 | } 87 | 88 | %{from: file, use_package_names: use_package_names, inject: true} -> 89 | only = last_module(namespace) 90 | 91 | %Config{ 92 | namespace: namespace, 93 | only: [only], 94 | from_file: file, 95 | inject: true, 96 | use_package_names: use_package_names, 97 | doc: doc 98 | } 99 | 100 | %{from: file, use_package_names: use_package_names} -> 101 | %Config{ 102 | namespace: namespace, 103 | from_file: file, 104 | doc: doc, 105 | use_package_names: use_package_names, 106 | use_google_types: use_google_types, 107 | } 108 | 109 | %{from: file, only: only, inject: true} -> 110 | types = parse_only(only, __CALLER__) 111 | 112 | case types do 113 | [] -> 114 | raise ConfigError, 115 | error: "You must specify a type using :only when combined with inject: true" 116 | 117 | [_type] -> 118 | %Config{ 119 | namespace: namespace, 120 | only: types, 121 | inject: true, 122 | from_file: file, 123 | doc: doc, 124 | use_google_types: use_google_types, 125 | } 126 | end 127 | 128 | %{from: file, only: only} -> 129 | %Config{ 130 | namespace: namespace, 131 | only: parse_only(only, __CALLER__), 132 | from_file: file, 133 | doc: doc, 134 | use_google_types: use_google_types, 135 | } 136 | 137 | %{from: file, inject: true} -> 138 | only = last_module(namespace) 139 | %Config{ 140 | namespace: namespace, 141 | only: [only], 142 | inject: true, 143 | from_file: file, 144 | doc: doc, 145 | use_google_types: use_google_types, 146 | } 147 | 148 | %{from: file} -> 149 | %Config{ 150 | namespace: namespace, 151 | from_file: file, 152 | doc: doc, 153 | use_google_types: use_google_types, 154 | } 155 | end 156 | 157 | config 158 | |> parse(__CALLER__) 159 | |> Builder.define(config) 160 | end 161 | 162 | # Read the type or list of types to extract from the schema 163 | defp parse_only(only, caller) do 164 | {types, []} = Code.eval_quoted(only, [], caller) 165 | 166 | case types do 167 | types when is_list(types) -> types 168 | types when types == nil -> [] 169 | _ -> [types] 170 | end 171 | end 172 | 173 | # Parse and fix namespaces of parsed types 174 | defp parse(%Config{schema: schema, inject: inject, from_file: nil} = config, caller) do 175 | ns = config.namespace 176 | mod_name = 177 | if inject do 178 | caller.module 179 | else 180 | ns 181 | end 182 | 183 | mod_name 184 | |> Parser.parse_string!(schema) 185 | |> namespace_types(ns, inject) 186 | end 187 | 188 | defp parse(%Config{inject: inject, from_file: file} = config, caller) do 189 | ns = config.namespace 190 | use_package_names = config.use_package_names 191 | {paths, import_dirs} = resolve_paths(config, file, caller) 192 | 193 | paths 194 | |> Parser.parse_files!(imports: import_dirs, use_packages: use_package_names) 195 | |> namespace_types(ns, inject) 196 | end 197 | 198 | # Apply namespace to top-level types 199 | defp namespace_types(parsed, ns, inject) do 200 | for {{type, name}, fields} <- parsed, is_atom(name) do 201 | parsed_type = if :gpb.is_msg_proto3(name, parsed), do: :proto3_msg, else: type 202 | 203 | if inject do 204 | ns_init = drop_last_module(ns) 205 | {{parsed_type, :"#{ns_init}.#{normalize_name(name)}"}, namespace_fields(type, fields, ns, true)} 206 | else 207 | {{parsed_type, :"#{ns}.#{normalize_name(name)}"}, namespace_fields(type, fields, ns, false)} 208 | end 209 | end 210 | end 211 | 212 | # Apply namespace to nested types 213 | defp namespace_fields(:msg, fields, ns, inject), do: Enum.map(fields, &namespace_fields(&1, ns, inject)) 214 | defp namespace_fields(:proto3_msg, fields, ns, inject), do: Enum.map(fields, &namespace_fields(&1, ns, inject)) 215 | defp namespace_fields(_, fields, _, _), do: fields 216 | 217 | defp namespace_fields(field, ns, inject) when not is_map(field) do 218 | case elem(field, 0) do 219 | :gpb_oneof -> 220 | field 221 | |> Utils.convert_from_record(OneOfField) 222 | |> namespace_fields(ns, inject) 223 | 224 | _ -> 225 | field 226 | |> Utils.convert_from_record(Field) 227 | |> namespace_fields(ns, inject) 228 | end 229 | end 230 | 231 | defp namespace_fields(%Field{type: {:map, key_type, value_type}} = field, ns, _inject) do 232 | key_type = namespace_map_type(key_type, ns) 233 | value_type = namespace_map_type(value_type, ns) 234 | %{field | type: {:map, key_type, value_type}} 235 | end 236 | 237 | defp namespace_fields(%Field{type: {type, name}} = field, ns, inject) do 238 | field_ns = if inject, do: drop_last_module(ns), else: ns 239 | %{field | :type => {type, :"#{field_ns}.#{normalize_name(name)}"}} 240 | end 241 | 242 | defp namespace_fields(%Field{} = field, _ns, _inject) do 243 | field 244 | end 245 | 246 | defp namespace_fields(%OneOfField{} = field, ns, inject) do 247 | fields = Enum.map(field.fields, &namespace_fields(&1, ns, inject)) 248 | Map.put(field, :fields, fields) 249 | end 250 | 251 | defp namespace_map_type({:msg, name}, ns) do 252 | {:msg, :"#{ns}.#{normalize_name(name)}"} 253 | end 254 | 255 | defp namespace_map_type(type, _ns) do 256 | type 257 | end 258 | 259 | # Normalizes module names by ensuring they are cased correctly 260 | # (respects camel-case and nested modules) 261 | defp normalize_name(name) do 262 | name 263 | |> Atom.to_string() 264 | |> String.split(".", parts: :infinity) 265 | |> Enum.map(fn x -> String.split_at(x, 1) end) 266 | |> Enum.map(fn {first, remainder} -> String.upcase(first) <> remainder end) 267 | |> Enum.join(".") 268 | |> String.to_atom() 269 | end 270 | 271 | defp resolve_paths(%Config{use_google_types: google_types?}, quoted_files, caller) do 272 | google_proto = Path.join(Application.app_dir(:exprotobuf, "priv"), "google_protobuf.proto") 273 | 274 | paths = 275 | case Code.eval_quoted(quoted_files, [], caller) do 276 | {path, _} when is_binary(path) and google_types? -> 277 | [google_proto, path] 278 | 279 | {path, _} when is_binary(path) -> 280 | [path] 281 | 282 | {paths, _} when is_list(paths) and google_types? -> 283 | [google_proto | paths] 284 | 285 | {paths, _} when is_list(paths) -> 286 | paths 287 | end 288 | 289 | import_dirs = 290 | paths 291 | |> Enum.map(&Path.dirname/1) 292 | |> Enum.uniq() 293 | 294 | {paths, import_dirs} 295 | end 296 | 297 | # Returns the last module of a namespace 298 | defp last_module(namespace) do 299 | namespace 300 | |> Module.split() 301 | |> List.last() 302 | |> String.to_atom() 303 | end 304 | 305 | defp drop_last_module(namespace) do 306 | namespace 307 | |> Atom.to_string() 308 | |> String.split(".") 309 | |> Enum.drop(-1) 310 | |> Enum.join(".") 311 | |> String.to_atom() 312 | end 313 | end 314 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, and 10 | distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by the copyright 13 | owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all other entities 16 | that control, are controlled by, or are under common control with that entity. 17 | For the purposes of this definition, "control" means (i) the power, direct or 18 | indirect, to cause the direction or management of such entity, whether by 19 | contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the 20 | outstanding shares, or (iii) beneficial ownership of such entity. 21 | 22 | "You" (or "Your") shall mean an individual or Legal Entity exercising 23 | permissions granted by this License. 24 | 25 | "Source" form shall mean the preferred form for making modifications, including 26 | but not limited to software source code, documentation source, and configuration 27 | files. 28 | 29 | "Object" form shall mean any form resulting from mechanical transformation or 30 | translation of a Source form, including but not limited to compiled object code, 31 | generated documentation, and conversions to other media types. 32 | 33 | "Work" shall mean the work of authorship, whether in Source or Object form, made 34 | available under the License, as indicated by a copyright notice that is included 35 | in or attached to the work (an example is provided in the Appendix below). 36 | 37 | "Derivative Works" shall mean any work, whether in Source or Object form, that 38 | is based on (or derived from) the Work and for which the editorial revisions, 39 | annotations, elaborations, or other modifications represent, as a whole, an 40 | original work of authorship. For the purposes of this License, Derivative Works 41 | shall not include works that remain separable from, or merely link (or bind by 42 | name) to the interfaces of, the Work and Derivative Works thereof. 43 | 44 | "Contribution" shall mean any work of authorship, including the original version 45 | of the Work and any modifications or additions to that Work or Derivative Works 46 | thereof, that is intentionally submitted to Licensor for inclusion in the Work 47 | by the copyright owner or by an individual or Legal Entity authorized to submit 48 | on behalf of the copyright owner. For the purposes of this definition, 49 | "submitted" means any form of electronic, verbal, or written communication sent 50 | to the Licensor or its representatives, including but not limited to 51 | communication on electronic mailing lists, source code control systems, and 52 | issue tracking systems that are managed by, or on behalf of, the Licensor for 53 | the purpose of discussing and improving the Work, but excluding communication 54 | that is conspicuously marked or otherwise designated in writing by the copyright 55 | owner as "Not a Contribution." 56 | 57 | "Contributor" shall mean Licensor and any individual or Legal Entity on behalf 58 | of whom a Contribution has been received by Licensor and subsequently 59 | incorporated within the Work. 60 | 61 | 2. Grant of Copyright License. 62 | 63 | Subject to the terms and conditions of this License, each Contributor hereby 64 | grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, 65 | irrevocable copyright license to reproduce, prepare Derivative Works of, 66 | publicly display, publicly perform, sublicense, and distribute the Work and such 67 | Derivative Works in Source or Object form. 68 | 69 | 3. Grant of Patent License. 70 | 71 | Subject to the terms and conditions of this License, each Contributor hereby 72 | grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, 73 | irrevocable (except as stated in this section) patent license to make, have 74 | made, use, offer to sell, sell, import, and otherwise transfer the Work, where 75 | such license applies only to those patent claims licensable by such Contributor 76 | that are necessarily infringed by their Contribution(s) alone or by combination 77 | of their Contribution(s) with the Work to which such Contribution(s) was 78 | submitted. If You institute patent litigation against any entity (including a 79 | cross-claim or counterclaim in a lawsuit) alleging that the Work or a 80 | Contribution incorporated within the Work constitutes direct or contributory 81 | patent infringement, then any patent licenses granted to You under this License 82 | for that Work shall terminate as of the date such litigation is filed. 83 | 84 | 4. Redistribution. 85 | 86 | You may reproduce and distribute copies of the Work or Derivative Works thereof 87 | in any medium, with or without modifications, and in Source or Object form, 88 | provided that You meet the following conditions: 89 | 90 | You must give any other recipients of the Work or Derivative Works a copy of 91 | this License; and 92 | You must cause any modified files to carry prominent notices stating that You 93 | changed the files; and 94 | You must retain, in the Source form of any Derivative Works that You distribute, 95 | all copyright, patent, trademark, and attribution notices from the Source form 96 | of the Work, excluding those notices that do not pertain to any part of the 97 | Derivative Works; and 98 | If the Work includes a "NOTICE" text file as part of its distribution, then any 99 | Derivative Works that You distribute must include a readable copy of the 100 | attribution notices contained within such NOTICE file, excluding those notices 101 | that do not pertain to any part of the Derivative Works, in at least one of the 102 | following places: within a NOTICE text file distributed as part of the 103 | Derivative Works; within the Source form or documentation, if provided along 104 | with the Derivative Works; or, within a display generated by the Derivative 105 | Works, if and wherever such third-party notices normally appear. The contents of 106 | the NOTICE file are for informational purposes only and do not modify the 107 | License. You may add Your own attribution notices within Derivative Works that 108 | You distribute, alongside or as an addendum to the NOTICE text from the Work, 109 | provided that such additional attribution notices cannot be construed as 110 | modifying the License. 111 | You may add Your own copyright statement to Your modifications and may provide 112 | additional or different license terms and conditions for use, reproduction, or 113 | distribution of Your modifications, or for any such Derivative Works as a whole, 114 | provided Your use, reproduction, and distribution of the Work otherwise complies 115 | with the conditions stated in this License. 116 | 117 | 5. Submission of Contributions. 118 | 119 | Unless You explicitly state otherwise, any Contribution intentionally submitted 120 | for inclusion in the Work by You to the Licensor shall be under the terms and 121 | conditions of this License, without any additional terms or conditions. 122 | Notwithstanding the above, nothing herein shall supersede or modify the terms of 123 | any separate license agreement you may have executed with Licensor regarding 124 | such Contributions. 125 | 126 | 6. Trademarks. 127 | 128 | This License does not grant permission to use the trade names, trademarks, 129 | service marks, or product names of the Licensor, except as required for 130 | reasonable and customary use in describing the origin of the Work and 131 | reproducing the content of the NOTICE file. 132 | 133 | 7. Disclaimer of Warranty. 134 | 135 | Unless required by applicable law or agreed to in writing, Licensor provides the 136 | Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, 137 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, 138 | including, without limitation, any warranties or conditions of TITLE, 139 | NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are 140 | solely responsible for determining the appropriateness of using or 141 | redistributing the Work and assume any risks associated with Your exercise of 142 | permissions under this License. 143 | 144 | 8. Limitation of Liability. 145 | 146 | In no event and under no legal theory, whether in tort (including negligence), 147 | contract, or otherwise, unless required by applicable law (such as deliberate 148 | and grossly negligent acts) or agreed to in writing, shall any Contributor be 149 | liable to You for damages, including any direct, indirect, special, incidental, 150 | or consequential damages of any character arising as a result of this License or 151 | out of the use or inability to use the Work (including but not limited to 152 | damages for loss of goodwill, work stoppage, computer failure or malfunction, or 153 | any and all other commercial damages or losses), even if such Contributor has 154 | been advised of the possibility of such damages. 155 | 156 | 9. Accepting Warranty or Additional Liability. 157 | 158 | While redistributing the Work or Derivative Works thereof, You may choose to 159 | offer, and charge a fee for, acceptance of support, warranty, indemnity, or 160 | other liability obligations and/or rights consistent with this License. However, 161 | in accepting such obligations, You may act only on Your own behalf and on Your 162 | sole responsibility, not on behalf of any other Contributor, and only if You 163 | agree to indemnify, defend, and hold each Contributor harmless for any liability 164 | incurred by, or claims asserted against, such Contributor by reason of your 165 | accepting any such warranty or additional liability. 166 | 167 | END OF TERMS AND CONDITIONS 168 | 169 | APPENDIX: How to apply the Apache License to your work 170 | 171 | To apply the Apache License to your work, attach the following boilerplate 172 | notice, with the fields enclosed by brackets "{}" replaced with your own 173 | identifying information. (Don't include the brackets!) The text should be 174 | enclosed in the appropriate comment syntax for the file format. We also 175 | recommend that a file or class name and description of purpose be included on 176 | the same "printed page" as the copyright notice for easier identification within 177 | third-party archives. 178 | 179 | Copyright 2013 Azuki Serviços de Internet LTDA. 180 | 181 | Licensed under the Apache License, Version 2.0 (the "License"); 182 | you may not use this file except in compliance with the License. 183 | You may obtain a copy of the License at 184 | 185 | http://www.apache.org/licenses/LICENSE-2.0 186 | 187 | Unless required by applicable law or agreed to in writing, software 188 | distributed under the License is distributed on an "AS IS" BASIS, 189 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 190 | See the License for the specific language governing permissions and 191 | limitations under the License. 192 | 193 | -------------------------------------------------------------------------------- /lib/exprotobuf/define_message.ex: -------------------------------------------------------------------------------- 1 | defmodule Protobuf.DefineMessage do 2 | @moduledoc false 3 | 4 | alias Protobuf.Decoder 5 | alias Protobuf.Encoder 6 | alias Protobuf.Field 7 | alias Protobuf.OneOfField 8 | alias Protobuf.Delimited 9 | alias Protobuf.Utils 10 | 11 | def def_message(name, fields, inject: inject, doc: doc, syntax: syntax) when is_list(fields) do 12 | struct_fields = record_fields(fields) 13 | # Inject everything in 'using' module 14 | if inject do 15 | quote location: :keep do 16 | @root __MODULE__ 17 | @record unquote(struct_fields) 18 | defstruct @record 19 | fields = unquote(struct_fields) 20 | 21 | def record, do: @record 22 | def syntax, do: unquote(syntax) 23 | 24 | unquote(define_typespec(name, fields)) 25 | 26 | unquote(encode_decode(name)) 27 | unquote(fields_methods(fields)) 28 | unquote(oneof_fields_methods(fields)) 29 | unquote(meta_information()) 30 | unquote(constructors(name)) 31 | 32 | defimpl Protobuf.Serializable do 33 | def serialize(object), do: unquote(name).encode(object) 34 | end 35 | end 36 | 37 | # Or create a nested module, with use_in functionality 38 | else 39 | quote location: :keep do 40 | root = __MODULE__ 41 | fields = unquote(struct_fields) 42 | use_in = @use_in[unquote(name)] 43 | 44 | defmodule unquote(name) do 45 | @moduledoc false 46 | unquote(Protobuf.Config.doc_quote(doc)) 47 | @root root 48 | @record unquote(struct_fields) 49 | defstruct @record 50 | 51 | def record, do: @record 52 | def syntax, do: unquote(syntax) 53 | 54 | unquote(define_typespec(name, fields)) 55 | 56 | unquote(encode_decode(name)) 57 | unquote(fields_methods(fields)) 58 | unquote(oneof_fields_methods(fields)) 59 | unquote(meta_information()) 60 | 61 | unquote(constructors(name)) 62 | 63 | if use_in != nil do 64 | Module.eval_quoted(__MODULE__, use_in, [], __ENV__) 65 | end 66 | 67 | defimpl Protobuf.Serializable do 68 | def serialize(object), do: unquote(name).encode(object) 69 | end 70 | end 71 | 72 | unquote(define_oneof_modules(name, fields)) 73 | end 74 | end 75 | end 76 | 77 | defp constructors(name) do 78 | quote location: :keep do 79 | def new(), do: new([]) 80 | 81 | def new(values) do 82 | struct(unquote(name), values) 83 | end 84 | end 85 | end 86 | 87 | defp define_typespec(module, field_list) when is_list(field_list) when is_atom(module) do 88 | case field_list do 89 | [%Field{name: :value, type: scalar, occurrence: occurrence}] 90 | when is_atom(scalar) and is_atom(occurrence) -> 91 | scalar_wrapper? = Utils.is_standard_scalar_wrapper(module) 92 | 93 | cond do 94 | scalar_wrapper? and occurrence == :required -> 95 | quote do 96 | @type t() :: unquote(define_scalar_typespec(scalar)) 97 | end 98 | 99 | scalar_wrapper? -> 100 | quote do 101 | @type t() :: unquote(define_scalar_typespec(scalar)) | nil 102 | end 103 | 104 | :else -> 105 | define_trivial_typespec(field_list) 106 | end 107 | 108 | [%Field{name: :value, type: {:enum, enum_module}, occurrence: occurrence}] 109 | when is_atom(enum_module) -> 110 | enum_wrapper? = Utils.is_enum_wrapper(module, enum_module) 111 | 112 | cond do 113 | enum_wrapper? and occurrence == :required -> 114 | quote do 115 | @type t() :: unquote(enum_module).t() 116 | end 117 | 118 | enum_wrapper? -> 119 | quote do 120 | @type t() :: unquote(enum_module).t() | nil 121 | end 122 | 123 | :else -> 124 | define_trivial_typespec(field_list) 125 | end 126 | 127 | _ -> 128 | define_trivial_typespec(field_list) 129 | end 130 | end 131 | 132 | defp define_trivial_typespec([]), do: nil 133 | 134 | defp define_trivial_typespec(fields) when is_list(fields) do 135 | field_types = define_trivial_typespec_fields(fields, []) 136 | map_type = {:%{}, [], field_types} 137 | module_type = {:%, [], [{:__MODULE__, [], Elixir}, map_type]} 138 | 139 | quote generated: true do 140 | @type t() :: unquote(module_type) 141 | end 142 | end 143 | 144 | defp define_trivial_typespec_fields([], acc), do: Enum.reverse(acc) 145 | 146 | defp define_trivial_typespec_fields([field | rest], acc) do 147 | case field do 148 | %Protobuf.Field{name: name, occurrence: :required, type: type} -> 149 | ast = {name, define_field_typespec(type)} 150 | define_trivial_typespec_fields(rest, [ast | acc]) 151 | 152 | %Protobuf.Field{name: name, occurrence: :optional, type: type} -> 153 | ast = 154 | {name, 155 | quote do 156 | unquote(define_field_typespec(type)) | nil 157 | end} 158 | 159 | define_trivial_typespec_fields(rest, [ast | acc]) 160 | 161 | %Protobuf.Field{name: name, occurrence: :repeated, type: type} -> 162 | ast = 163 | {name, 164 | quote do 165 | [unquote(define_field_typespec(type))] 166 | end} 167 | 168 | define_trivial_typespec_fields(rest, [ast | acc]) 169 | 170 | %Protobuf.OneOfField{name: name, fields: fields} -> 171 | ast = 172 | {name, 173 | quote do 174 | unquote(define_algebraic_type(fields)) 175 | end} 176 | 177 | define_trivial_typespec_fields(rest, [ast | acc]) 178 | end 179 | end 180 | 181 | defp define_algebraic_type(fields) do 182 | ast = 183 | for %Protobuf.Field{name: name, type: type} <- fields do 184 | {name, define_field_typespec(type)} 185 | end 186 | 187 | Protobuf.Utils.define_algebraic_type([nil | ast]) 188 | end 189 | 190 | defp define_oneof_modules(namespace, fields) when is_list(fields) do 191 | ast = 192 | for %Protobuf.OneOfField{} = field <- fields do 193 | define_oneof_instance_module(namespace, field) 194 | end 195 | 196 | quote do 197 | (unquote_splicing(ast)) 198 | end 199 | end 200 | 201 | defp define_oneof_instance_module(namespace, %Protobuf.OneOfField{name: field, fields: fields}) do 202 | module_subname = 203 | field 204 | |> Atom.to_string() 205 | |> Macro.camelize() 206 | |> String.to_atom() 207 | 208 | fields = Enum.map(fields, &define_oneof_instance_macro/1) 209 | 210 | quote do 211 | defmodule unquote(Module.concat([namespace, :OneOf, module_subname])) do 212 | (unquote_splicing(fields)) 213 | end 214 | end 215 | end 216 | 217 | defp define_oneof_instance_macro(%Protobuf.Field{name: name}) do 218 | quote do 219 | defmacro unquote(name)(ast) do 220 | inner_name = unquote(name) 221 | 222 | quote do 223 | {unquote(inner_name), unquote(ast)} 224 | end 225 | end 226 | end 227 | end 228 | 229 | defp define_field_typespec(type) do 230 | case type do 231 | {:msg, field_module} -> 232 | quote do 233 | unquote(field_module).t() 234 | end 235 | 236 | {:enum, field_module} -> 237 | quote do 238 | unquote(field_module).t() 239 | end 240 | 241 | {:map, key_type, value_type} -> 242 | key_type_ast = define_field_typespec(key_type) 243 | value_type_ast = define_field_typespec(value_type) 244 | 245 | quote do 246 | [{unquote(key_type_ast), unquote(value_type_ast)}] 247 | end 248 | 249 | _ -> 250 | define_scalar_typespec(type) 251 | end 252 | end 253 | 254 | defp define_scalar_typespec(type) do 255 | case type do 256 | :double -> 257 | quote do 258 | float() 259 | end 260 | 261 | :float -> 262 | quote do 263 | float() 264 | end 265 | 266 | :int32 -> 267 | quote do 268 | integer() 269 | end 270 | 271 | :int64 -> 272 | quote do 273 | integer() 274 | end 275 | 276 | :uint32 -> 277 | quote do 278 | non_neg_integer() 279 | end 280 | 281 | :uint64 -> 282 | quote do 283 | non_neg_integer() 284 | end 285 | 286 | :sint32 -> 287 | quote do 288 | integer() 289 | end 290 | 291 | :sint64 -> 292 | quote do 293 | integer() 294 | end 295 | 296 | :fixed32 -> 297 | quote do 298 | non_neg_integer() 299 | end 300 | 301 | :fixed64 -> 302 | quote do 303 | non_neg_integer() 304 | end 305 | 306 | :sfixed32 -> 307 | quote do 308 | integer() 309 | end 310 | 311 | :sfixed64 -> 312 | quote do 313 | integer() 314 | end 315 | 316 | :bool -> 317 | quote do 318 | boolean() 319 | end 320 | 321 | :string -> 322 | quote do 323 | String.t() 324 | end 325 | 326 | :bytes -> 327 | quote do 328 | binary() 329 | end 330 | end 331 | end 332 | 333 | defp encode_decode(_name) do 334 | quote do 335 | def decode(data), do: Decoder.decode(data, __MODULE__) 336 | def encode(%{} = record), do: Encoder.encode(record, defs()) 337 | def decode_delimited(bytes), do: Delimited.decode(bytes, __MODULE__) 338 | def encode_delimited(messages), do: Delimited.encode(messages) 339 | end 340 | end 341 | 342 | defp fields_methods(fields) do 343 | for %Field{name: name, fnum: fnum} = field <- fields do 344 | quote location: :keep do 345 | def defs(:field, unquote(fnum)), do: unquote(Macro.escape(field)) 346 | def defs(:field, unquote(name)), do: defs(:field, unquote(fnum)) 347 | end 348 | end 349 | end 350 | 351 | defp oneof_fields_methods(fields) do 352 | for %OneOfField{name: name, rnum: rnum} = field <- fields do 353 | quote location: :keep do 354 | def defs(:field, unquote(rnum - 1)), do: unquote(Macro.escape(field)) 355 | def defs(:field, unquote(name)), do: defs(:field, unquote(rnum - 1)) 356 | end 357 | end 358 | end 359 | 360 | defp meta_information do 361 | quote do 362 | def defs, do: @root.defs 363 | def defs(:field, _), do: nil 364 | def defs(:field, field, _), do: defs(:field, field) 365 | defoverridable defs: 0 366 | end 367 | end 368 | 369 | defp record_fields(fields) do 370 | fields 371 | |> Enum.map(fn field -> 372 | case field do 373 | %Field{name: name, occurrence: :repeated} -> 374 | {name, []} 375 | 376 | %Field{name: name, opts: [default: default]} -> 377 | {name, default} 378 | 379 | %Field{name: name} -> 380 | {name, nil} 381 | 382 | %OneOfField{name: name} -> 383 | {name, nil} 384 | 385 | _ -> 386 | nil 387 | end 388 | end) 389 | |> Enum.reject(&is_nil/1) 390 | end 391 | end 392 | -------------------------------------------------------------------------------- /test/protobuf_test.exs: -------------------------------------------------------------------------------- 1 | defmodule ProtobufTest do 2 | use Protobuf.Case 3 | 4 | test "can roundtrip encoding/decoding optional values in proto2" do 5 | defmodule RoundtripProto2 do 6 | use Protobuf, """ 7 | message Msg { 8 | optional string f1 = 1; 9 | optional string f2 = 2 [default = "test"]; 10 | optional uint32 f3 = 3; 11 | oneof f4 { 12 | string f4a = 4; 13 | } 14 | } 15 | """ 16 | end 17 | 18 | msg1 = RoundtripProto2.Msg.new() 19 | encoded1 = RoundtripProto2.Msg.encode(msg1) 20 | assert %{f1: nil, f2: "test", f3: nil, f4: nil} = RoundtripProto2.Msg.decode(encoded1) 21 | 22 | msg2 = RoundtripProto2.Msg.new(f4: {:f4a, "test"}) 23 | encoded2 = RoundtripProto2.Msg.encode(msg2) 24 | assert %{f4: {:f4a, "test"}} = RoundtripProto2.Msg.decode(encoded2) 25 | end 26 | 27 | test "can roundtrip encoding/decoding optional values in proto3" do 28 | defmodule RoundtripProto3 do 29 | use Protobuf, """ 30 | syntax = "proto3"; 31 | 32 | message Msg { 33 | string f1 = 1; 34 | uint32 f2 = 2; 35 | bool f3 = 3; 36 | oneof f4 { 37 | string f4a = 4; 38 | } 39 | } 40 | """ 41 | end 42 | 43 | msg1 = RoundtripProto3.Msg.new() 44 | encoded1 = RoundtripProto3.Msg.encode(msg1) 45 | assert %{f1: "", f2: 0, f3: false, f4: nil} = RoundtripProto3.Msg.decode(encoded1) 46 | 47 | msg2 = RoundtripProto3.Msg.new(f4: {:f4a, "test"}) 48 | encoded2 = RoundtripProto3.Msg.encode(msg2) 49 | assert %{f4: {:f4a, "test"}} = RoundtripProto3.Msg.decode(encoded2) 50 | end 51 | 52 | test "can encode when protocol is extended with new optional field" do 53 | defmodule BasicProto do 54 | use Protobuf, """ 55 | message Msg { 56 | required uint32 f1 = 1; 57 | } 58 | """ 59 | end 60 | old = BasicProto.Msg.new(f1: 1) 61 | 62 | defmodule BasicProto do 63 | use Protobuf, """ 64 | message Msg { 65 | required uint32 f1 = 1; 66 | optional uint32 f2 = 2; 67 | } 68 | """ 69 | end 70 | encoded = BasicProto.Msg.encode(old) 71 | decoded = BasicProto.Msg.decode(encoded) 72 | 73 | assert 1 = decoded.f1 74 | refute decoded.f2 75 | end 76 | 77 | test "can encode when inject is used" do 78 | defmodule Msg do 79 | use Protobuf, [""" 80 | message Msg { 81 | required uint32 f1 = 1; 82 | } 83 | """, inject: true] 84 | end 85 | msg = Msg.new(f1: 1) 86 | encoded = Msg.encode(msg) 87 | decoded = Msg.decode(encoded) 88 | 89 | assert 1 = decoded.f1 90 | end 91 | 92 | test "can encode when inject and only are used" do 93 | defmodule Msg do 94 | use Protobuf, [""" 95 | message Msg { 96 | required uint32 f1 = 1; 97 | } 98 | """, inject: true, only: [:Msg]] 99 | end 100 | msg = Msg.new(f1: 1) 101 | encoded = Msg.encode(msg) 102 | decoded = Msg.decode(encoded) 103 | 104 | assert 1 = decoded.f1 105 | end 106 | 107 | test "can encode when inject is used and module is nested" do 108 | defmodule Nested.Msg do 109 | use Protobuf, [""" 110 | message Msg { 111 | required uint32 f1 = 1; 112 | } 113 | """, inject: true] 114 | end 115 | msg = Nested.Msg.new(f1: 1) 116 | encoded = Nested.Msg.encode(msg) 117 | decoded = Nested.Msg.decode(encoded) 118 | 119 | assert 1 = decoded.f1 120 | end 121 | 122 | test "can encode when inject is used and definition loaded from a file" do 123 | defmodule Basic do 124 | use Protobuf, from: Path.expand("./proto/simple.proto", __DIR__), inject: true 125 | end 126 | basic = Basic.new(f1: 1) 127 | encoded = Basic.encode(basic) 128 | decoded = Basic.decode(encoded) 129 | assert 1 == decoded.f1 130 | end 131 | 132 | test "can decode when protocol is extended with new optional field" do 133 | defmodule BasicProto do 134 | use Protobuf, """ 135 | message Msg { 136 | required uint32 f1 = 1; 137 | } 138 | """ 139 | end 140 | old = BasicProto.Msg.new(f1: 1) 141 | encoded = BasicProto.Msg.encode(old) 142 | 143 | defmodule BasicProto do 144 | use Protobuf, """ 145 | message Msg { 146 | required uint32 f1 = 1; 147 | optional uint32 f2 = 2; 148 | } 149 | """ 150 | end 151 | decoded = BasicProto.Msg.decode(encoded) 152 | 153 | assert 1 = decoded.f1 154 | refute decoded.f2 155 | end 156 | 157 | test "define records in namespace" do 158 | defmodule NamespacedRecordsProto do 159 | use Protobuf, """ 160 | message Msg1 { 161 | required uint32 f1 = 1; 162 | } 163 | 164 | message Msg2 { 165 | required string f1 = 1; 166 | } 167 | """ 168 | end 169 | msg1 = NamespacedRecordsProto.Msg1 170 | msg2 = NamespacedRecordsProto.Msg2 171 | 172 | assert %{:__struct__ => ^msg1, :f1 => 1} = NamespacedRecordsProto.Msg1.new(f1: 1) 173 | assert %{:__struct__ => ^msg2, :f1 => "foo"} = NamespacedRecordsProto.Msg2.new(f1: "foo") 174 | end 175 | 176 | test "define records in namespace with injection" do 177 | contents = quote do 178 | use Protobuf, [" 179 | message InjectionTest { 180 | required uint32 f1 = 1; 181 | } 182 | ", inject: true] 183 | end 184 | 185 | {:module, mod, _, _} = Module.create(InjectionTest, contents, Macro.Env.location(__ENV__)) 186 | 187 | assert %{:__struct__ => ^mod, :f1 => 1} = mod.new(f1: 1) 188 | end 189 | 190 | test "namespaces of not injected modules are valid with inject" do 191 | contents = quote do 192 | use Protobuf, [" 193 | message A { 194 | required uint32 f1 = 1; 195 | optional B b = 2; 196 | } 197 | 198 | message B { 199 | required uint32 fB = 1; 200 | } 201 | ", inject: true] 202 | end 203 | 204 | {:module, mod, _, _} = Module.create(A, contents, Macro.Env.location(__ENV__)) 205 | def_a = Enum.find(mod.defs, &match?({{:msg, A}, _}, &1)) 206 | {:msg, ns_field_b} = def_a |> elem(1) |> Enum.at(1) |> Map.get(:type) 207 | assert ns_field_b == B 208 | end 209 | 210 | test "allow inject, use_package_names and from_file at the same time" do 211 | defmodule Version do 212 | use Protobuf, 213 | from: Path.expand("./proto/mumble.proto", __DIR__), 214 | inject: true, 215 | use_package_names: true 216 | end 217 | 218 | assert Keyword.has_key?(Version.__info__(:functions), :new) 219 | end 220 | 221 | test "do not set default value for optional" do 222 | defmodule DefaultValueForOptionalsProto do 223 | use Protobuf, "message Msg { optional uint32 f1 = 1; }" 224 | end 225 | msg = DefaultValueForOptionalsProto.Msg.new() 226 | assert nil == msg.f1 227 | end 228 | 229 | test "set default value to [] for repeated" do 230 | defmodule DefaultValueForListsProto do 231 | use Protobuf, "message Msg { repeated uint32 f1 = 1; }" 232 | end 233 | msg = DefaultValueForListsProto.Msg.new() 234 | assert [] == msg.f1 235 | end 236 | 237 | test "set default value if specified explicitly" do 238 | defmodule DefaultValueExplicitProto do 239 | use Protobuf, "message Msg { optional uint32 f1 = 1 [default = 42]; }" 240 | end 241 | msg = DefaultValueExplicitProto.Msg.new() 242 | assert 42 == msg.f1 243 | end 244 | 245 | test "does not set default value if there is a type mismatch" do 246 | assert_raise Protobuf.Parser.ParserError, fn -> 247 | defmodule InvalidValueDefaultValueExplicitProto do 248 | use Protobuf, "message Msg { optional uint32 f1 = 1 [default = -1]; }" 249 | end 250 | end 251 | end 252 | 253 | test "define a record in subnamespace" do 254 | defmodule SubnamespacedRecordProto do 255 | use Protobuf, """ 256 | message Msg { 257 | message SubMsg { 258 | required uint32 f1 = 1; 259 | } 260 | 261 | required SubMsg f1 = 1; 262 | } 263 | """ 264 | end 265 | 266 | msg = SubnamespacedRecordProto.Msg.SubMsg.new(f1: 1) 267 | module = SubnamespacedRecordProto.Msg.SubMsg 268 | assert %{__struct__: ^module} = msg 269 | 270 | msg = SubnamespacedRecordProto.Msg.new(f1: msg) 271 | assert %{__struct__: ^module} = msg.f1 272 | end 273 | 274 | test "define enum information module" do 275 | defmodule EnumInfoModProto do 276 | use Protobuf, """ 277 | enum Version { 278 | V0_1 = 1; 279 | V0_2 = 2; 280 | } 281 | message Msg { 282 | enum MsgType { 283 | START = 1; 284 | STOP = 2; 285 | } 286 | required MsgType type = 1; 287 | required Version version = 2; 288 | } 289 | """ 290 | end 291 | 292 | assert {:file, []} == :code.is_loaded(EnumInfoModProto.Version) 293 | assert {:file, []} == :code.is_loaded(EnumInfoModProto.Msg.MsgType) 294 | 295 | assert 1 == EnumInfoModProto.Version.value(:V0_1) 296 | assert 1 == EnumInfoModProto.Msg.MsgType.value(:START) 297 | 298 | assert :V0_2 == EnumInfoModProto.Version.atom(2) 299 | assert :STOP == EnumInfoModProto.Msg.MsgType.atom(2) 300 | 301 | assert nil == EnumInfoModProto.Version.atom(-1) 302 | assert nil == EnumInfoModProto.Msg.MsgType.value(:OTHER) 303 | 304 | assert [:V0_1, :V0_2] == EnumInfoModProto.Version.atoms 305 | assert [:START, :STOP] == EnumInfoModProto.Msg.MsgType.atoms 306 | 307 | assert [1, 2] == EnumInfoModProto.Version.values 308 | assert [1, 2] == EnumInfoModProto.Msg.MsgType.values 309 | end 310 | 311 | test "support define from a file" do 312 | defmodule ProtoFromFile do 313 | use Protobuf, from: Path.expand("./proto/basic.proto", __DIR__) 314 | end 315 | 316 | basic = ProtoFromFile.Basic.new(f1: 1) 317 | module = ProtoFromFile.Basic 318 | assert %{__struct__: ^module} = basic 319 | end 320 | 321 | test "define a method to get proto defs" do 322 | defmodule ProtoDefsProto do 323 | use Protobuf, "message Msg { optional uint32 f1 = 1; }" 324 | end 325 | defs = [{{:msg, ProtoDefsProto.Msg}, [%Protobuf.Field{name: :f1, fnum: 1, rnum: 2, type: :uint32, occurrence: :optional, opts: []}]}] 326 | assert defs == ProtoDefsProto.defs 327 | assert defs == ProtoDefsProto.Msg.defs 328 | end 329 | 330 | test "defined a method defs to get field info" do 331 | defmodule FieldDefsProto do 332 | use Protobuf, "message Msg { optional uint32 f1 = 1; }" 333 | end 334 | deff = %Protobuf.Field{name: :f1, fnum: 1, rnum: 2, type: :uint32, occurrence: :optional, opts: []} 335 | assert deff == FieldDefsProto.Msg.defs(:field, 1) 336 | assert deff == FieldDefsProto.Msg.defs(:field, :f1) 337 | end 338 | 339 | test "defined method decode" do 340 | defmodule DecodeMethodProto do 341 | use Protobuf, "message Msg { optional uint32 f1 = 1; }" 342 | end 343 | module = DecodeMethodProto.Msg 344 | assert %{:__struct__ => ^module} = DecodeMethodProto.Msg.decode(<<>>) 345 | end 346 | 347 | test "extensions skip" do 348 | defmodule SkipExtensions do 349 | use Protobuf, """ 350 | message Msg { 351 | required uint32 f1 = 1; 352 | extensions 100 to 200; 353 | } 354 | """ 355 | end 356 | module = SkipExtensions.Msg 357 | assert %{:__struct__ => ^module} = SkipExtensions.Msg.new 358 | end 359 | 360 | test "additional method via use_in" do 361 | defmodule AddViaHelper do 362 | use Protobuf, "message Msg { 363 | required uint32 f1 = 1; 364 | }" 365 | 366 | defmodule MsgHelper do 367 | defmacro __using__(_opts) do 368 | quote do 369 | def sub(%{:f1 => f1} = msg, value) do 370 | %{msg | :f1 => f1 - value} 371 | end 372 | end 373 | end 374 | end 375 | 376 | use_in :Msg, MsgHelper 377 | end 378 | 379 | msg = AddViaHelper.Msg.new(f1: 10) 380 | assert %{:f1 => 5} = AddViaHelper.Msg.sub(msg, 5) 381 | end 382 | 383 | test "normalize unconventional (lowercase) styles of named messages and enums" do 384 | defmodule UnconventionalMessagesProto do 385 | use Protobuf, """ 386 | message msgPackage { 387 | enum msgResponseType { 388 | NACK = 0; 389 | ACK = 1; 390 | } 391 | 392 | message msgHeader { 393 | required uint32 message_id = 0; 394 | required msgResponseType response_type = 1; 395 | } 396 | 397 | required msgHeader header = 1; 398 | } 399 | """ 400 | end 401 | 402 | assert 0 == UnconventionalMessagesProto.MsgPackage.MsgResponseType.value(:NACK) 403 | assert 1 == UnconventionalMessagesProto.MsgPackage.MsgResponseType.value(:ACK) 404 | 405 | assert :NACK == UnconventionalMessagesProto.MsgPackage.MsgResponseType.atom(0) 406 | assert :ACK == UnconventionalMessagesProto.MsgPackage.MsgResponseType.atom(1) 407 | 408 | msg_header = UnconventionalMessagesProto.MsgPackage.MsgHeader 409 | 410 | assert %{:__struct__ => ^msg_header, 411 | :response_type => :ACK, 412 | :message_id => 25 } = UnconventionalMessagesProto.MsgPackage.MsgHeader.new(response_type: :ACK, message_id: 25) 413 | 414 | msg_package = UnconventionalMessagesProto.MsgPackage 415 | nack_msg_header = UnconventionalMessagesProto.MsgPackage.MsgHeader.new(message_id: 1, response_type: :NACK) 416 | #nack_msg_package = UnconventionalMessagesProto.MsgPackage.new(header: nack_msg_header) 417 | 418 | assert %{:__struct__ => ^msg_package, :header => ^nack_msg_header} = UnconventionalMessagesProto.MsgPackage.new(header: nack_msg_header) 419 | end 420 | end 421 | --------------------------------------------------------------------------------