├── .gitignore ├── README.md ├── lib ├── compile.exprotoc.ex ├── exprotoc.ex └── exprotoc │ ├── ast.ex │ ├── generator.ex │ ├── message.ex │ ├── parser.ex │ └── protocol.ex ├── mix.exs ├── src └── proto_grammar.yrl └── test └── test_wrapper ├── .gitignore ├── README.md ├── mix.exs ├── priv ├── another.proto ├── nopackage.proto ├── other.proto └── test.proto └── test ├── test_helper.exs └── test_wrapper_test.exs /.gitignore: -------------------------------------------------------------------------------- 1 | /_build 2 | /deps 3 | erl_crash.dump 4 | *.ez 5 | proto_grammar.erl 6 | test/test_wrapper/lib -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Exprotoc 2 | 3 | Elixir protocol buffers compiler 4 | 5 | ## Mix project options 6 | 7 | To use `exprotoc`, first include `exprotoc` in your mixfile as a 8 | dependency. Then, in the project property of your mix project, add 9 | `:exprotoc` to the list of compilers (e.g. `compilers: 10 | [:exprotoc, :elixir, :app`). 11 | 12 | To configure the `:exprotoc` compiler prepass, there are three 13 | exposed options: `proto_out`, `proto_files`, and `proto_path`. 14 | 15 | ### `proto_out` 16 | 17 | Binary string that represents the directory generated code should 18 | be output to. Defaults to the `lib` folder. 19 | 20 | ### `proto_files` 21 | 22 | This should be a list of all the proto files you wish to turn into 23 | elixir code. 24 | 25 | ### `proto_path` 26 | 27 | List of directories in the order `exprotoc` should visit to look 28 | for proto files and imports. 29 | 30 | ### Example `mix.exs file` 31 | 32 | ```elixir 33 | defmodule Example.Mixfile do 34 | use Mix.Project 35 | 36 | def project do 37 | [ app: :example, 38 | version: "0.0.1", 39 | elixir: "~> 0.12.5", 40 | compilers: [:exprotoc, :elixir, :app], 41 | proto_files: ["example.proto"], 42 | proto_path: ["priv"], 43 | deps: deps ] 44 | end 45 | 46 | defp deps do 47 | [{ :exprotoc, github: "jeremyong/exprotoc" }] 48 | end 49 | end 50 | ``` 51 | 52 | This will compile the `Example` application with the generated code 53 | for the proto file `example.proto`. Because no `proto_out` option 54 | is specified, the generated code will be output to the lib folder. 55 | 56 | ## Usage 57 | 58 | Just access your message modules as you would a standard Elixir dict. 59 | 60 | For example, with the following proto message: 61 | 62 | ```proto 63 | package Example; 64 | message Foo { 65 | enum Bar { 66 | Zap = 150; 67 | } 68 | required Foo a = 1; 69 | required uint32 b = 2; 70 | } 71 | ``` 72 | 73 | `exprotoc` will create a module `Example.Foo` and submodule 74 | `Example.Foo.Bar`. 75 | 76 | You can create a new message simply with `Example.Foo.new` or 77 | `Example.Foo.new a: Example.Foo.Bar.zap`. 78 | 79 | You can access your message with the access protocol. For example: 80 | 81 | ```elixir 82 | message = Example.Foo.new a: Example.Foo.Bar.zap 83 | IO.inspect message[:a] # Will output { Example.Foo.Bar, :zap } 84 | IO.inspect message[:b] # Will output nil 85 | message = Example.Foo.put message, :b, 150 86 | IO.inspect message[:b] # Will output 150 87 | ``` 88 | 89 | All message modules export the `encode` and `decode` functions to turn 90 | a message into an iolist or turn binary into a message. 91 | 92 | Default values work as expected. 93 | 94 | Repeated fields are represented as lists. 95 | -------------------------------------------------------------------------------- /lib/compile.exprotoc.ex: -------------------------------------------------------------------------------- 1 | defmodule Mix.Tasks.Compile.Exprotoc do 2 | use Mix.Task 3 | 4 | def run(_) do 5 | { :ok, out_dir } = get_out_dir 6 | File.mkdir_p out_dir 7 | { :ok, proto_files } = Keyword.fetch Mix.project, :proto_files 8 | { :ok, proto_path } = get_path 9 | Enum.each proto_files, &Exprotoc.compile(&1, out_dir, proto_path) 10 | end 11 | 12 | defp get_out_dir do 13 | path = Keyword.fetch Mix.project, :proto_out 14 | if path == :error do 15 | { :ok, cwd } = File.cwd 16 | path = Path.join cwd, "lib" 17 | { :ok, path } 18 | else 19 | path 20 | end 21 | end 22 | 23 | defp get_path do 24 | path = Keyword.fetch Mix.project, :proto_path 25 | if path == :error do 26 | { :ok, [] } 27 | else 28 | path 29 | end 30 | end 31 | end -------------------------------------------------------------------------------- /lib/exprotoc.ex: -------------------------------------------------------------------------------- 1 | defmodule Exprotoc do 2 | import Exprotoc.Parser 3 | import Exprotoc.AST 4 | def compile(file, out_dir, proto_path) do 5 | ast = file |> tokenize(proto_path) |> parse |> generate_ast(proto_path) 6 | Exprotoc.Generator.generate_code ast, out_dir 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /lib/exprotoc/ast.ex: -------------------------------------------------------------------------------- 1 | defmodule Exprotoc.AST do 2 | import Exprotoc.Parser 3 | 4 | @moduledoc "Generates a structured AST from the AST produced by the parser." 5 | 6 | def generate_ast({ :no_package, imports, { enums, messages } }, proto_path) do 7 | ast = generate_symbols enums, messages, HashDict.new 8 | full_ast = generate_import_ast imports, proto_path, ast 9 | { ast, full_ast } 10 | end 11 | def generate_ast({ { :package, package }, imports, { enums, messages } }, 12 | proto_path) do 13 | ast = generate_symbols enums, messages, HashDict.new 14 | ast = HashDict.put HashDict.new, package, { [], ast } 15 | full_ast = generate_import_ast imports, proto_path, ast 16 | { package, ast, full_ast } 17 | end 18 | 19 | defp generate_import_ast([], _, acc), do: acc 20 | defp generate_import_ast([ i | imports ], proto_path, acc) do 21 | file = find_file i, proto_path 22 | { package, _, { enums, messages } } = file |> tokenize(proto_path) |> parse 23 | ast = generate_symbols enums, messages, HashDict.new 24 | acc = merge_asts { package, ast }, acc 25 | generate_import_ast imports, proto_path, acc 26 | end 27 | 28 | defp merge_asts({ :nopackage, modules }, ast) do 29 | HashDict.merge modules, ast, fn(k, _, _) -> 30 | raise "Ambiguous name for #{k}." 31 | end 32 | end 33 | defp merge_asts({ { :package, package }, modules }, ast) do 34 | if HashDict.has_key? ast, package do 35 | raise "Ambiguous name for #{package}." 36 | end 37 | HashDict.put ast, package, { [], modules } 38 | end 39 | 40 | def search_ast(ast, [], needle) do 41 | module = traverse_ast ast, needle 42 | if module == nil do 43 | name = Exprotoc.Generator.get_module_name needle 44 | raise "Could not identify symbol for #{name}." 45 | else 46 | { module, needle } 47 | end 48 | end 49 | def search_ast(ast, inner = [_|outer], needle) do 50 | reversed_scope = Enum.reverse inner 51 | branch = traverse_ast ast, reversed_scope 52 | module = traverse_ast branch, needle 53 | if module == nil do 54 | search_ast ast, outer, needle 55 | else 56 | { module, reversed_scope ++ needle } 57 | end 58 | end 59 | 60 | def traverse_ast(_, []), do: nil 61 | def traverse_ast({ _, ast }, [pointer]) do 62 | if HashDict.has_key?(ast, pointer) do 63 | ast[pointer] 64 | else 65 | nil 66 | end 67 | end 68 | def traverse_ast({ _, ast }, [pointer|pointers]) do 69 | if HashDict.has_key?(ast, pointer) do 70 | traverse_ast ast[pointer], pointers 71 | else 72 | nil 73 | end 74 | end 75 | 76 | defp generate_symbols(enums, messages, symbol_tree) do 77 | symbol_tree = List.foldl enums, symbol_tree, &add_enum/2 78 | List.foldl messages, symbol_tree, &add_message/2 79 | end 80 | 81 | defp add_enum({ :enum, enum, enum_data }, symbol_tree) do 82 | if nil? symbol_tree[enum] do 83 | symbol_tree = HashDict.put symbol_tree, enum, { :enum, enum_data } 84 | else 85 | raise "Duplicate symbol for enum #{enum}." 86 | end 87 | symbol_tree 88 | end 89 | 90 | defp add_message({ :message, message, { enums, messages, fields } }, 91 | symbol_tree) do 92 | if nil? symbol_tree[message] do 93 | subtree = HashDict.new 94 | subtree = generate_symbols(enums, messages, subtree) 95 | symbol_tree = HashDict.put symbol_tree, message, {fields, subtree} 96 | else 97 | raise "Duplicate symbol for message #{message}." 98 | end 99 | symbol_tree 100 | end 101 | end -------------------------------------------------------------------------------- /lib/exprotoc/generator.ex: -------------------------------------------------------------------------------- 1 | defmodule Exprotoc.Generator do 2 | @moduledoc "Given a structured AST, generate code files into an output dir." 3 | 4 | def generate_code({ ast, full_ast }, dir) do 5 | File.mkdir_p dir 6 | generate_modules { [], full_ast }, [], HashDict.to_list(ast), dir 7 | end 8 | def generate_code({ package, ast, full_ast}, dir) do 9 | dir = create_package_dir package, dir 10 | { [], ast } = ast[package] 11 | generate_modules { [], full_ast }, [package], HashDict.to_list(ast), dir 12 | end 13 | 14 | defp create_package_dir(package, dir) do 15 | package_name = to_enum_type package 16 | dir = dir |> Path.join(package_name) 17 | File.mkdir_p dir 18 | dir 19 | end 20 | 21 | defp generate_modules(_, _, [], _) do end 22 | defp generate_modules(ast, scope, [module|modules], dir) do 23 | { name, _ } = module 24 | new_scope = [ name | scope ] 25 | module_filename = name |> atom_to_binary |> to_module_filename 26 | path = Path.join dir, module_filename 27 | IO.puts "Generating #{module_filename}" 28 | module_text = generate_module ast, new_scope, module, 0, false 29 | File.write path, module_text 30 | generate_modules ast, scope, modules, dir 31 | end 32 | 33 | defp generate_module(_, scope, { _, { :enum, enum_values } }, level, sub) do 34 | fullname = scope |> Enum.reverse |> get_module_name 35 | if sub do 36 | [name|_] = scope 37 | name = atom_to_binary name 38 | else 39 | name = fullname 40 | end 41 | i = indent level 42 | { acc1, acc2, acc3 } = 43 | List.foldl enum_values, { "", "", "" }, 44 | fn({k, v}, { a1, a2, a3 }) -> 45 | enum_atom = to_enum_type k 46 | a1 = a1 <> 47 | "#{i} def to_i({ #{fullname}, :#{enum_atom} }), do: #{v}\n" 48 | a2 = a2 <> 49 | "#{i} def to_symbol(#{v}), do: { #{fullname}, :#{enum_atom} }\n" 50 | a3 = a3 <> 51 | "#{i} def #{enum_atom}, do: { #{fullname}, :#{enum_atom} }\n" 52 | { a1, a2, a3 } 53 | end 54 | enum_funs = acc1 <> acc2 <> acc3 55 | """ 56 | #{i}defmodule #{name} do 57 | #{i} def decode(value), do: to_symbol value 58 | #{enum_funs}#{i}end 59 | """ 60 | end 61 | defp generate_module(ast, scope, module, level, sub) do 62 | if sub do 63 | [name|_] = scope 64 | name = atom_to_binary name 65 | else 66 | name = scope |> Enum.reverse |> get_module_name 67 | end 68 | i = indent level 69 | fields_text = process_fields ast, scope, module, level 70 | submodules = module |> elem(1) |> elem(1) |> HashDict.to_list 71 | submodule_text = List.foldl submodules, "", 72 | fn(m = { n, _ }, acc) -> 73 | acc <> generate_module(ast, [n|scope], 74 | m, level + 1, true) 75 | end 76 | 77 | """ 78 | #{i}defmodule #{name} do 79 | #{i} defrecord T, message: HashDict.new 80 | #{i} def encode(msg) do 81 | #{i} p = List.foldl get_keys, [], fn(key, acc) -> 82 | #{i} fnum = get_fnum key 83 | #{i} type = get_type fnum 84 | #{i} value = msg.message[fnum] 85 | #{i} if value == nil do 86 | #{i} value = get_default fnum 87 | #{i} end 88 | #{i} if value == nil do 89 | #{i} if get_ftype(fnum) == :required do 90 | #{i} raise \"Missing field \#{key} in encoding __MODULE__\" 91 | #{i} end 92 | #{i} acc 93 | #{i} else 94 | #{i} [ { fnum, { type, value } } | acc ] 95 | #{i} end 96 | #{i} end 97 | #{i} Exprotoc.Protocol.encode_message p 98 | #{i} end 99 | 100 | #{i} def decode(payload) do 101 | #{i} m = Exprotoc.Protocol.decode_payload payload, __MODULE__ 102 | #{i} T.new message: m 103 | #{i} end 104 | 105 | #{i} def new, do: T.new 106 | #{i} def new(enum) do 107 | #{i} new T.new, enum 108 | #{i} end 109 | #{i} def new(msg, enum) do 110 | #{i} Enum.reduce enum, msg, fn({k, v}, acc) -> 111 | #{i} put acc, k, v 112 | #{i} end 113 | #{i} end 114 | #{i} def get(msg, key) do 115 | #{i} f_num = get_fnum key 116 | #{i} m = msg.message 117 | #{i} if HashDict.has_key?(m, f_num) do 118 | #{i} if get_ftype(f_num) == :repeated do 119 | #{i} elem m[f_num], 1 120 | #{i} else 121 | #{i} m[f_num] 122 | #{i} end 123 | #{i} else 124 | #{i} if get_ftype(f_num) == :repeated do 125 | #{i} [] 126 | #{i} else 127 | #{i} get_default f_num 128 | #{i} end 129 | #{i} end 130 | #{i} end 131 | #{i} def put(msg, key, value) do 132 | #{i} f_num = get_fnum key 133 | #{i} m = msg.message 134 | #{i} m = put_key m, f_num, value 135 | #{i} msg.message m 136 | #{i} end 137 | #{i} def delete(msg, key) do 138 | #{i} f_num = get_fnum key 139 | #{i} m = msg.message 140 | #{i} m = HashDict.delete m, f_num 141 | #{i} msg.message m 142 | #{i} end 143 | 144 | #{fields_text}#{submodule_text}#{i}end 145 | 146 | #{i}defimpl Access, for: #{name}.T do 147 | #{i} def access(msg, key), do: #{name}.get(msg, key) 148 | #{i}end 149 | """ 150 | end 151 | 152 | def get_module_name(names) do 153 | Enum.join names, "." 154 | end 155 | 156 | defp process_fields(ast, scope, { _, { fields, _ } }, level) do 157 | process_fields ast, scope, fields, level 158 | end 159 | defp process_fields(ast, scope, fields, level) do 160 | i = indent level + 1 161 | acc = { "", "", "", "", "", [] } 162 | { acc1, acc2, acc3, acc4, acc5, acc6 } = 163 | List.foldl fields, acc, 164 | &process_field(ast, scope, &1, &2, i) 165 | acc5 = acc5 <> "#{i}def get_default(_), do: nil\n" 166 | key_string = generate_keystring acc6, i 167 | acc1 <> acc2 <> acc3 <> acc4 <> acc5 <> key_string 168 | end 169 | 170 | defp generate_keystring(keys, i) do 171 | keys = Enum.map keys, fn(key) -> ":" <> atom_to_binary(key) end 172 | center = Enum.join keys, ", " 173 | """ 174 | #{i}def get_keys, do: [ #{center} ] 175 | """ 176 | end 177 | 178 | defp process_field(ast, scope, { :field, ftype, type, name, fnum, opts }, 179 | { acc1, acc2, acc3, acc4, acc5, acc6 } , i) do 180 | type = type_to_string ast, scope, type 181 | if ftype == :repeated do 182 | acc1 = acc1 <> """ 183 | #{i}defp put_key(msg, #{fnum}, values) when is_list(values) do 184 | #{i} HashDict.put msg, #{fnum}, { :repeated, values } 185 | #{i}end 186 | """ 187 | else 188 | acc1 = acc1 <> """ 189 | #{i}defp put_key(msg, #{fnum}, value) do 190 | #{i} HashDict.put msg, #{fnum}, value 191 | #{i}end 192 | """ 193 | end 194 | acc2 = acc2 <> "#{i}def get_fnum(:#{name}), do: #{fnum}\n" 195 | acc3 = acc3 <> "#{i}def get_ftype(#{fnum}), do: :#{ftype}\n" 196 | acc4 = acc4 <> "#{i}def get_type(#{fnum}), do: #{type}\n" 197 | if opts[:default] != nil do 198 | acc5 = acc5 <> "#{i}def get_default(#{fnum}), do: #{opts[:default]}\n" 199 | end 200 | { acc1, acc2, acc3, acc4, acc5, [ name | acc6 ] } 201 | end 202 | 203 | defp indent(level), do: String.duplicate(" ", level) 204 | 205 | defp type_to_string(ast, scope, type) when is_list(type) do 206 | { module, pointer } = Exprotoc.AST.search_ast ast, scope, type 207 | if elem(module, 0) == :enum do 208 | "{ :enum, " <> get_module_name(pointer) <> " }" 209 | else 210 | "{ :message, " <> get_module_name(pointer) <> " }" 211 | end 212 | end 213 | defp type_to_string(ast, scope, type) do 214 | if Exprotoc.Protocol.wire_type(type) == :custom do 215 | type_to_string ast, scope, [type] 216 | else 217 | ":" <> atom_to_binary(type) 218 | end 219 | end 220 | 221 | defp to_module_filename(module) do 222 | module = String.downcase module 223 | Regex.replace(~r/\./, module, "_") <> ".ex" 224 | end 225 | 226 | defp to_enum_type(name) do 227 | name = atom_to_binary name 228 | name = Regex.replace ~r/([A-Z])/, name, "_\\1" 229 | name |> String.lstrip(?_) |> String.downcase 230 | end 231 | end -------------------------------------------------------------------------------- /lib/exprotoc/message.ex: -------------------------------------------------------------------------------- 1 | defmodule Exprotoc.Message do 2 | @type t :: Dict.t 3 | end -------------------------------------------------------------------------------- /lib/exprotoc/parser.ex: -------------------------------------------------------------------------------- 1 | defmodule Exprotoc.Parser do 2 | @multiline_comment "/\\*([^*]|\\*+[^*/])*\\*+/" 3 | @line_comment "//.*\\n" 4 | 5 | def tokenize(file, path) do 6 | file = find_file file, path 7 | { :ok, text } = File.read file 8 | { :ok, reg1 } = Regex.compile @multiline_comment 9 | { :ok, reg2 } = Regex.compile @line_comment 10 | text = Regex.replace reg1, text, "" 11 | text = Regex.replace reg2, text, "" 12 | { :ok, list_text } = String.to_char_list text 13 | { :ok, tokens, _ } = :erl_scan.string(list_text, 1, 14 | { :reserved_word_fun , 15 | &reserved_words/1 }) 16 | tokens 17 | end 18 | 19 | def find_file(file, []) do 20 | if File.exists? file do 21 | file 22 | else 23 | raise "Could not locate #{file} in path" 24 | end 25 | end 26 | def find_file(file, [ dir | proto_path ]) do 27 | file_path = Path.join dir, file 28 | if File.exists? file_path do 29 | file_path 30 | else 31 | find_file file, proto_path 32 | end 33 | end 34 | 35 | def parse(tokens) do 36 | { :ok, ast } = :proto_grammar.parse tokens 37 | ast 38 | end 39 | 40 | defp reserved_words(:package), do: true 41 | defp reserved_words(:message), do: true 42 | defp reserved_words(:enum), do: true 43 | defp reserved_words(:packed), do: true 44 | defp reserved_words(:default), do: true 45 | defp reserved_words(true), do: true 46 | defp reserved_words(false), do: true 47 | defp reserved_words(:import), do: true 48 | defp reserved_words(_), do: false 49 | end -------------------------------------------------------------------------------- /lib/exprotoc/protocol.ex: -------------------------------------------------------------------------------- 1 | defmodule Exprotoc.Protocol do 2 | use Bitwise 3 | 4 | @type wire_type :: 0 | 1 | 2 | 5 5 | @type value :: integer | float | binary 6 | 7 | @spec decode_payload(binary | list, atom) :: Message.t 8 | def decode_payload(message, module) when is_list(message) do 9 | decode_payload iolist_to_binary(message), module 10 | end 11 | def decode_payload(payload, module) do 12 | { message, keys } = decode_payload payload, module, HashDict.new, [] 13 | List.foldl keys, message, 14 | fn(k, m) -> 15 | { :repeated, vs } = m[k] 16 | HashDict.put m, k, { :repeated, Enum.reverse(vs) } 17 | end 18 | end 19 | 20 | defp decode_payload("", _, acc, keys), do: { acc, keys } 21 | defp decode_payload(message, module, acc, keys) do 22 | { varint, message } = pop_varint message 23 | field_num = varint >>> 3 24 | wire_type = varint - (field_num <<< 3) 25 | { value, message } = pop_value(wire_type, message) 26 | field_type = module.get_ftype field_num 27 | data_type = module.get_type field_num 28 | value = cast value, data_type 29 | if field_type == :repeated do 30 | if HashDict.has_key? acc, field_num do 31 | { :repeated, current } = acc[field_num] 32 | acc = HashDict.put acc, field_num , { :repeated, [ value | current ] } 33 | else 34 | acc = HashDict.put acc, field_num, { :repeated, [value] } 35 | keys = [ field_num | keys ] 36 | end 37 | else 38 | acc = HashDict.put acc, field_num, value 39 | end 40 | decode_payload message, module, acc, keys 41 | end 42 | 43 | def encode_message(message) do 44 | message = message |> List.keysort(1) |> Enum.reverse 45 | encode_message message, [] 46 | end 47 | 48 | defp encode_message([], acc), do: acc 49 | defp encode_message([ { field_num, { type, { :repeated, values } } } | rest ], 50 | acc) do 51 | payload = Enum.map values, &encode_value(field_num, type, &1) 52 | encode_message rest, [ payload | acc ] 53 | end 54 | defp encode_message([ { field_num, { type, value } } | rest ], 55 | acc) do 56 | payload = encode_value field_num, type, value 57 | encode_message rest, [ payload | acc ] 58 | end 59 | 60 | @spec pop_value(wire_type, binary) :: { value, binary } 61 | defp pop_value(0, message), do: pop_varint(message) 62 | defp pop_value(1, message), do: pop_64bits(message) 63 | defp pop_value(2, message), do: pop_string(message) 64 | defp pop_value(5, message), do: pop_32bits(message) 65 | 66 | defp pop_varint(message) do 67 | pop_varint(message, 0, 0) 68 | end 69 | defp pop_varint(<< 1 :: 1, data :: 7, rest :: binary >>, 70 | acc, pad) do 71 | pop_varint rest, (data <<< pad) + acc, pad + 7 72 | end 73 | defp pop_varint(<< 0 :: 1, data:: 7, rest :: binary >>, 74 | acc, pad) do 75 | { (data <<< pad) + acc, rest } 76 | end 77 | 78 | defp pop_64bits(<< value :: [64, unit(1), binary], rest :: binary >>), do: { value, rest } 79 | 80 | defp pop_32bits(<< value :: [32, unit(1), binary], rest :: binary >>), do: { value, rest } 81 | 82 | defp pop_string(message) do 83 | { len, message } = pop_varint message 84 | << string :: [ size(len), binary ], message :: binary >> = message 85 | { string, message } 86 | end 87 | 88 | defp encode_value(_, _, :undefined), do: [] 89 | defp encode_value(field_num, { :enum, enum }, value) do 90 | varint = enum.to_i value 91 | [ encode_varint(field_num <<< 3), encode_varint(varint) ] 92 | end 93 | defp encode_value(field_num, { :message, module }, message) do 94 | payload = module.encode message 95 | size = iolist_size payload 96 | [ encode_varint((field_num <<< 3) ||| 2), encode_varint(size) , payload ] 97 | end 98 | defp encode_value(field_num, type, data) do 99 | key = (field_num <<< 3) ||| wire_type(type) 100 | [ key, encode_value(type, data) ] 101 | end 102 | 103 | defp encode_value(:int32, data) when data < 0 do 104 | encode_varint(data + (1 <<< 32)) 105 | end 106 | defp encode_value(:int32, data) do 107 | encode_varint data 108 | end 109 | defp encode_value(:int64, data) when data < 0 do 110 | encode_varint(data + (1 <<< 64)) 111 | end 112 | defp encode_value(:int64, data) do 113 | encode_varint data 114 | end 115 | defp encode_value(:uint32, data), do: encode_varint(data) 116 | defp encode_value(:uint64, data), do: encode_varint(data) 117 | defp encode_value(:sint32, data) 118 | when data <= 0x80000000 119 | when data >= -0x7fffffff do 120 | int = bxor (data <<< 1), (data >>> 31) 121 | encode_varint int 122 | end 123 | defp encode_value(:sint64, data) 124 | when data <= 0x8000000000000000 125 | when data >= -0x7fffffffffffffff do 126 | int = bxor (data <<< 1), (data >>> 63) 127 | encode_varint int 128 | end 129 | defp encode_value(:bool, true), do: encode_varint(1) 130 | defp encode_value(:bool, false), do: encode_varint(0) 131 | defp encode_value(:string, data), do: encode_value(:bytes, data) 132 | defp encode_value(:bytes, data) do 133 | len = byte_size data 134 | [ encode_varint(len), data ] 135 | end 136 | defp encode_value(:float, data) do 137 | << data :: [ size(32), float, little ] >> 138 | end 139 | 140 | defp encode_varint(data) when data >= 0 do 141 | data |> encode_varint([]) |> Enum.reverse 142 | end 143 | defp encode_varint(true, acc), do: [1|acc] 144 | defp encode_varint(false, acc), do: [0|acc] 145 | defp encode_varint(int, acc) when int <= 127, do: [int|acc] 146 | defp encode_varint(int, acc) do 147 | next = int >>> 7 148 | last_seven = int - (next <<< 7) 149 | acc = [ (1 <<< 7) + last_seven | acc ] 150 | encode_varint next, acc 151 | end 152 | 153 | def wire_type(:int32), do: 0 154 | def wire_type(:int64), do: 0 155 | def wire_type(:uint32), do: 0 156 | def wire_type(:uint64), do: 0 157 | def wire_type(:sint32), do: 0 158 | def wire_type(:sint64), do: 0 159 | def wire_type(:bool), do: 0 160 | def wire_type(:enum), do: 0 161 | def wire_type(:fixed64), do: 1 162 | def wire_type(:sfixed64), do: 1 163 | def wire_type(:double), do: 1 164 | def wire_type(:string), do: 2 165 | def wire_type(:bytes), do: 2 166 | def wire_type(:embedded), do: 2 167 | def wire_type(:repeated), do: 2 168 | def wire_type(:fixed32), do: 5 169 | def wire_type(:sfixed32), do: 5 170 | def wire_type(:float), do: 5 171 | def wire_type(_), do: :custom 172 | 173 | defp cast(value, :int32) do 174 | if value &&& 0x8000000000000000 != 0 do 175 | value - 0x8000000000000000 176 | else 177 | value 178 | end 179 | end 180 | defp cast(value, :int64) do 181 | if value &&& 0x8000000000000000 != 0 do 182 | value - 0x8000000000000000 183 | else 184 | value 185 | end 186 | end 187 | defp cast(value, :uint32), do: value 188 | defp cast(value, :uint64), do: value 189 | defp cast(value, :sint32) do 190 | bxor (value >>> 1), -(value &&& 1) 191 | end 192 | defp cast(value, :sint64) do 193 | bxor (value >>> 1), -(value &&& 1) 194 | end 195 | defp cast(value, :string), do: value 196 | defp cast(value, :bytes), do: value 197 | defp cast(1, :bool), do: true 198 | defp cast(0, :bool), do: false 199 | defp cast(<< value :: [ size(32), little, unsigned, integer ] >>, 200 | :fixed32), do: value 201 | defp cast(<< value :: [ size(32), little, unsigned, integer ] >>, 202 | :sfixed32) do 203 | bxor (value >>> 1), -(value &&& 1) 204 | end 205 | defp cast(<< value :: [ size(64), little, unsigned, integer ] >>, 206 | :fixed64), do: value 207 | defp cast(<< value :: [ size(64), little, unsigned, integer ] >>, 208 | :sfixed64) do 209 | bxor (value >>> 1), -(value &&& 1) 210 | end 211 | defp cast(<< value :: [ little, float ] >>, :double), do: value 212 | defp cast(value, :float) do 213 | bits = byte_size(value) * 8 214 | << float :: [ size(bits), little, float ] >> = value 215 | float 216 | end 217 | defp cast(value, { :enum, enum }) do 218 | enum.to_symbol value 219 | end 220 | defp cast(value, { :message, module }) do 221 | module.decode value 222 | end 223 | 224 | end 225 | -------------------------------------------------------------------------------- /mix.exs: -------------------------------------------------------------------------------- 1 | defmodule Exprotoc.Mixfile do 2 | use Mix.Project 3 | 4 | def project do 5 | [ app: :exprotoc, 6 | version: "0.0.1", 7 | elixir: "~> 0.12.5", 8 | compilers: [ :yecc, :erlang, :elixir, :app ], 9 | deps: deps ] 10 | end 11 | 12 | # Configuration for the OTP application 13 | def application do 14 | [] 15 | end 16 | 17 | # Returns the list of dependencies in the format: 18 | # { :foobar, git: "https://github.com/elixir-lang/foobar.git", tag: "0.1" } 19 | # 20 | # To specify particular versions, regardless of the tag, do: 21 | # { :barbat, "~> 0.1", github: "elixir-lang/barbat" } 22 | defp deps do 23 | [] 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /src/proto_grammar.yrl: -------------------------------------------------------------------------------- 1 | %% Proto file yecc grammar 2 | 3 | Nonterminals 4 | proto imports messages 5 | p_enum enum_fields enum_field enum_name enum_value 6 | p_message message_name 7 | fields field field_rule field_num field_type field_name 8 | options option. 9 | 10 | Terminals 11 | ';' '=' '{' '}' '[' ']' ',' '.' 12 | package default packed message import 13 | enum atom string integer float var bool true false. 14 | 15 | Rootsymbol proto. 16 | 17 | proto -> package var ';' imports messages : { { package, value_of('$2') }, '$4', '$5'}. 18 | proto -> imports messages : { no_package, '$1', '$2' }. 19 | 20 | imports -> import string ';' imports : [list_to_binary(value_of('$2')) | '$4']. 21 | imports -> '$empty' : []. 22 | 23 | messages -> p_enum : {['$1'], []}. 24 | messages -> p_message : {[], ['$1']}. 25 | messages -> p_enum messages : add_first('$1', '$2'). 26 | messages -> p_message messages : add_second('$1', '$2'). 27 | 28 | p_message -> message message_name '{' fields '}' : {message, '$2', '$4'}. 29 | 30 | message_name -> var : value_of('$1'). 31 | 32 | fields -> p_enum : {['$1'], []}. 33 | fields -> p_enum fields : add_first_field('$1', '$2'). 34 | fields -> p_message : {[], ['$1'], []}. 35 | fields -> p_message fields : add_second_field('$1', '$2'). 36 | fields -> field : {[], [], ['$1']}. 37 | fields -> field fields : add_third_field('$1', '$2'). 38 | 39 | field -> 40 | field_rule field_type field_name '=' field_num '[' options ']' ';' 41 | : {field, '$1', '$2', '$3', '$5', '$7'}. 42 | field -> 43 | field_rule field_type field_name '=' field_num ';' 44 | : {field, '$1', '$2', '$3', '$5', []}. 45 | 46 | field_rule -> atom : value_of('$1'). 47 | 48 | field_type -> atom : value_of('$1'). 49 | field_type -> var : value_of('$1'). 50 | field_type -> var '.' field_type : prepend(value_of('$1'), '$3'). 51 | 52 | field_name -> atom : value_of('$1'). 53 | 54 | field_num -> integer : value_of('$1'). 55 | 56 | options -> option : ['$1']. 57 | options -> option ',' options : ['$1'|'$3']. 58 | 59 | option -> packed '=' true : {packed, true}. 60 | option -> packed '=' false : {packed, false}. 61 | option -> default '=' integer : {default, value_of('$3')}. 62 | option -> default '=' float : {default, value_of('$3')}. 63 | option -> default '=' string : {default, value_of('$3')}. 64 | option -> default '=' var : {default, value_of('$3')}. 65 | option -> default '=' bool : {default, value_of('$3')}. 66 | option -> default '=' true : {default, true}. 67 | option -> default '=' false : {default, false}. 68 | 69 | p_enum -> enum enum_name '{' enum_fields '}' : {enum, '$2', '$4'}. 70 | 71 | enum_fields -> enum_field : ['$1']. 72 | enum_fields -> enum_field enum_fields : ['$1'|'$2']. 73 | 74 | enum_field -> enum_name '=' enum_value ';' : {'$1', '$3'}. 75 | 76 | enum_name -> atom : value_of('$1'). 77 | enum_name -> var : value_of('$1'). 78 | 79 | enum_value -> integer : integer_to_list(value_of('$1')). 80 | 81 | Erlang code. 82 | add_first(A, {As, Bs}) -> 83 | {[A|As], Bs}. 84 | 85 | add_second(B, {As, Bs}) -> 86 | {As, [B|Bs]}. 87 | 88 | add_first_field(A, {As, Bs, Cs}) -> 89 | {[A|As], Bs, Cs}. 90 | 91 | add_second_field(B, {As, Bs, Cs}) -> 92 | {As, [B|Bs], Cs}. 93 | 94 | add_third_field(C, {As, Bs, Cs}) -> 95 | {As, Bs, [C|Cs]}. 96 | 97 | value_of(Token) -> 98 | element(3, Token). 99 | 100 | prepend(F, Fs) when is_list(Fs) -> 101 | [F|Fs]; 102 | prepend(F1, F2) -> 103 | [F1, F2]. 104 | -------------------------------------------------------------------------------- /test/test_wrapper/.gitignore: -------------------------------------------------------------------------------- 1 | /_build 2 | /deps 3 | erl_crash.dump 4 | *.ez 5 | -------------------------------------------------------------------------------- /test/test_wrapper/README.md: -------------------------------------------------------------------------------- 1 | # Tmp 2 | 3 | ** TODO: Add description ** 4 | -------------------------------------------------------------------------------- /test/test_wrapper/mix.exs: -------------------------------------------------------------------------------- 1 | defmodule TestWrapper.Mixfile do 2 | use Mix.Project 3 | 4 | def project do 5 | [ app: :test_wrapper, 6 | version: "0.0.1", 7 | elixir: "~> 0.12.5", 8 | compilers: [:exprotoc, :elixir, :app], 9 | proto_files: ["nopackage.proto", 10 | "test.proto", 11 | "other.proto", 12 | "another.proto"], 13 | proto_path: ["priv"], 14 | deps: deps ] 15 | end 16 | 17 | # Returns the list of dependencies in the format: 18 | # { :foobar, git: "https://github.com/elixir-lang/foobar.git", tag: "0.1" } 19 | # 20 | # To specify particular versions, regardless of the tag, do: 21 | # { :barbat, "~> 0.1", github: "elixir-lang/barbat" } 22 | defp deps do 23 | [ 24 | { :exprotoc, path: "../.." } 25 | ] 26 | end 27 | end 28 | -------------------------------------------------------------------------------- /test/test_wrapper/priv/another.proto: -------------------------------------------------------------------------------- 1 | package Another; 2 | 3 | message Msg2 { 4 | required uint32 a = 1; 5 | } 6 | -------------------------------------------------------------------------------- /test/test_wrapper/priv/nopackage.proto: -------------------------------------------------------------------------------- 1 | message NoPackage { 2 | required uint32 a = 1; 3 | } 4 | 5 | enum Multi { 6 | V1 = 1; 7 | V2 = 2; 8 | } -------------------------------------------------------------------------------- /test/test_wrapper/priv/other.proto: -------------------------------------------------------------------------------- 1 | package Other; 2 | 3 | import "another.proto"; 4 | 5 | message Msg1 { 6 | message Msg3 { 7 | required uint32 b = 1; 8 | } 9 | required uint32 a = 1; 10 | } 11 | -------------------------------------------------------------------------------- /test/test_wrapper/priv/test.proto: -------------------------------------------------------------------------------- 1 | package Test; 2 | 3 | import "other.proto"; 4 | import "another.proto"; 5 | 6 | message Test1 { 7 | required uint32 a = 1; 8 | } 9 | 10 | message Test2 { 11 | enum Foo { 12 | Bar = 150; 13 | } 14 | required Foo b = 1; 15 | } 16 | 17 | message Test3 { 18 | required Test1 c = 3; 19 | } 20 | 21 | message Test4 { 22 | repeated uint32 d = 1; 23 | } 24 | 25 | message Test5 { 26 | repeated Test6 e = 1; 27 | message Test6 { 28 | required uint32 f = 1; 29 | } 30 | } 31 | 32 | message Test7 { 33 | optional bool g = 1 [default = true]; 34 | } 35 | 36 | message Test8 { 37 | required uint32 h = 1; 38 | required uint32 i = 2; 39 | } 40 | 41 | message Test9 { 42 | required Test2.Foo j = 1; 43 | } 44 | 45 | message Test10 { 46 | required Test5.Test6 k = 1; 47 | } 48 | 49 | message Test11 { 50 | required Other.Msg1 l = 1; 51 | } 52 | 53 | message Test12 { 54 | required Other.Msg1.Msg3 m = 1; 55 | } 56 | 57 | message Test13 { 58 | optional uint32 n = 1 [default = 150]; 59 | optional uint32 o = 2 [default = 300]; 60 | } 61 | 62 | message Test14 { 63 | required float num = 1; 64 | } -------------------------------------------------------------------------------- /test/test_wrapper/test/test_helper.exs: -------------------------------------------------------------------------------- 1 | ExUnit.start 2 | -------------------------------------------------------------------------------- /test/test_wrapper/test/test_wrapper_test.exs: -------------------------------------------------------------------------------- 1 | defmodule TestWrapperTest do 2 | use ExUnit.Case 3 | 4 | test "access uint32" do 5 | t = Test.Test1.new a: 3 6 | assert t[:a] == 3 7 | t = Test.Test1.put t, :a, 4 8 | assert t[:a] == 4 9 | end 10 | 11 | test "encode uint32" do 12 | t = Test.Test1.new a: 150 13 | p = t |> Test.Test1.encode |> iolist_to_binary 14 | assert p == << 8, 150, 1 >> 15 | end 16 | 17 | test "decode uint32" do 18 | t = Test.Test1.decode << 8, 150, 1 >> 19 | assert t[:a] == 150 20 | end 21 | 22 | test "encode enum" do 23 | t = Test.Test2.new b: Test.Test2.Foo.bar 24 | assert t[:b] == Test.Test2.Foo.bar 25 | payload = t |> Test.Test2.encode |> iolist_to_binary 26 | assert payload == << 8, 150, 1 >> 27 | end 28 | 29 | test "decode enum" do 30 | payload = << 8, 150, 1>> 31 | message = Test.Test2.decode payload 32 | assert message[:b] == { Test.Test2.Foo, :bar } 33 | end 34 | 35 | test "encode nested message" do 36 | inner = Test.Test1.new a: 150 37 | outer = Test.Test3.new c: inner 38 | assert outer[:c][:a] == 150 39 | payload = outer |> Test.Test3.encode |> iolist_to_binary 40 | assert payload == << 26, 3, 8, 150, 1 >> 41 | end 42 | 43 | test "decode nested message" do 44 | payload = << 26, 3, 8, 150, 1 >> 45 | message = Test.Test3.decode payload 46 | assert message[:c][:a] == 150 47 | end 48 | 49 | test "encode repeated message" do 50 | message = Test.Test4.new d: [250, 150] 51 | assert message[:d] == [250, 150] 52 | payload = message |> Test.Test4.encode |> iolist_to_binary 53 | assert payload == << 8, 250, 1, 8, 150, 1 >> 54 | end 55 | 56 | test "decode repeated message" do 57 | payload = << 8, 250, 1, 8, 150, 1 >> 58 | message = Test.Test4.decode payload 59 | assert message[:d] == [250, 150] 60 | end 61 | 62 | test "encode nested repeated messages" do 63 | m1 = Test.Test5.Test6.new f: 150 64 | m2 = Test.Test5.Test6.new f: 300 65 | message = Test.Test5.new e: [m1, m2] 66 | payload = message |> Test.Test5.encode |> iolist_to_binary 67 | assert payload == << 10, 3, 8, 150, 1, 10, 3, 8, 172, 2 >> 68 | end 69 | 70 | test "decode nested repeated messages" do 71 | payload = << 10, 3, 8, 150, 1, 10, 3, 8, 172, 2 >> 72 | message = Test.Test5.decode payload 73 | [m1, m2] = message[:e] 74 | assert m1[:f] == 150 75 | assert m2[:f] == 300 76 | end 77 | 78 | test "encode bool" do 79 | message = Test.Test7.new g: true 80 | assert message[:g] == true 81 | payload = message |> Test.Test7.encode |> iolist_to_binary 82 | assert payload == << 8, 1 >> 83 | end 84 | 85 | test "decode bool" do 86 | payload = << 8, 1 >> 87 | message = Test.Test7.decode payload 88 | assert message[:g] == true 89 | payload = << 8, 0 >> 90 | message = Test.Test7.decode payload 91 | assert message[:g] == false 92 | end 93 | 94 | test "encode external enum" do 95 | message = Test.Test9.new j: Test.Test2.Foo.bar 96 | assert message[:j] == { Test.Test2.Foo, :bar } 97 | payload = message |> Test.Test9.encode |> iolist_to_binary 98 | assert payload == << 8, 150, 1 >> 99 | end 100 | 101 | test "decode external enum" do 102 | payload = << 8, 150, 1 >> 103 | message = Test.Test9.decode payload 104 | assert message[:j] == { Test.Test2.Foo, :bar } 105 | end 106 | 107 | test "encode external message" do 108 | m = Test.Test5.Test6.new f: 150 109 | message = Test.Test10.new k: m 110 | assert message[:k][:f] == 150 111 | payload = message |> Test.Test10.encode |> iolist_to_binary 112 | assert payload == << 10, 3, 8, 150, 1 >> 113 | end 114 | 115 | test "decode external message" do 116 | payload = << 10, 3, 8, 150, 1 >> 117 | message = Test.Test10.decode payload 118 | assert message[:k][:f] == 150 119 | end 120 | 121 | test "incode imported message" do 122 | m = Other.Msg1.new a: 150 123 | message = Test.Test11.new l: m 124 | assert message[:l][:a] == 150 125 | payload = message |> Test.Test11.encode |> iolist_to_binary 126 | assert payload == << 10, 3, 8, 150, 1 >> 127 | end 128 | 129 | test "decode imported message" do 130 | payload = << 10, 3, 8, 150, 1>> 131 | message = Test.Test11.decode payload 132 | assert message[:l][:a] == 150 133 | end 134 | 135 | test "encode nested imported message" do 136 | m = Other.Msg1.Msg3.new b: 150 137 | message = Test.Test12.new m: m 138 | assert message[:m][:b] == 150 139 | payload = message |> Test.Test12.encode |> iolist_to_binary 140 | assert payload == << 10, 3, 8, 150, 1 >> 141 | end 142 | 143 | test "decode nested imported message" do 144 | payload = << 10, 3, 8, 150, 1>> 145 | message = Test.Test12.decode payload 146 | assert message[:m][:b] == 150 147 | end 148 | 149 | test "missing package header" do 150 | message = NoPackage.new a: 150 151 | assert message[:a] == 150 152 | payload = message |> NoPackage.encode |> iolist_to_binary 153 | assert payload == << 8, 150, 1 >> 154 | message = NoPackage.decode payload 155 | assert message[:a] == 150 156 | end 157 | 158 | test "access field with default" do 159 | message = Test.Test7.new 160 | assert message[:g] == true 161 | end 162 | 163 | test "encode field with defaults" do 164 | message = Test.Test13.new 165 | assert message[:n] == 150 166 | assert message[:o] == 300 167 | payload = message |> Test.Test13.encode |> iolist_to_binary 168 | assert payload == << 8, 150, 1, 16, 172, 2 >> 169 | end 170 | 171 | test "decode field with defaults" do 172 | payload = <<>> 173 | message = Test.Test13.decode payload 174 | assert message[:n] == 150 175 | assert message[:o] == 300 176 | end 177 | 178 | test "copy existing message" do 179 | message = Test.Test13.new n: 100 180 | copy = Test.Test13.new message, o: 200 181 | assert copy[:n] == 100 182 | assert copy[:o] == 200 183 | end 184 | 185 | test "encode float" do 186 | t = Test.Test14.new num: 1.0 187 | p = t |> Test.Test14.encode |> iolist_to_binary 188 | assert p == <<13, 0, 0, 128, 63>> 189 | end 190 | 191 | test "decode float" do 192 | t = Test.Test14.decode <<13, 0, 0, 128, 63>> 193 | assert t[:num] == 1.0 194 | end 195 | end 196 | --------------------------------------------------------------------------------