├── .gitignore ├── test ├── test_helper.exs ├── eden_test.exs └── eden │ ├── parser_test.exs │ └── lexer_test.exs ├── LICENSE ├── Makefile ├── lib ├── eden │ ├── types.ex │ ├── parser │ │ └── node.ex │ ├── exception.ex │ ├── encode.ex │ ├── decode.ex │ ├── parser.ex │ └── lexer.ex └── eden.ex ├── .github └── workflows │ └── build.yml ├── config └── config.exs ├── mix.exs ├── README.md ├── mix.lock └── CHANGELOG.md /.gitignore: -------------------------------------------------------------------------------- 1 | /_build 2 | /deps 3 | erl_crash.dump 4 | *.ez 5 | cover/ 6 | doc/ -------------------------------------------------------------------------------- /test/test_helper.exs: -------------------------------------------------------------------------------- 1 | require Logger 2 | Logger.configure([level: :error]) 3 | 4 | ExUnit.start() 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2015 Juan Facorro 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PROJECT=Eden 2 | 3 | all: deps app protocols 4 | 5 | get-deps: 6 | rm -f mix.lock 7 | mix deps.get 8 | 9 | deps: get-deps 10 | mix deps.compile 11 | 12 | app: 13 | mix compile 14 | 15 | protocols: 16 | mix compile.protocols 17 | 18 | clean-deps: 19 | mix deps.clean --all 20 | rm -rf deps 21 | 22 | clean: clean-deps 23 | mix clean 24 | 25 | test: app 26 | mix test 27 | 28 | shell: app 29 | iex --name ${PROJECT}@`hostname` -pa _build/dev/consolidated -S mix 30 | 31 | escript: 32 | mix escript.build 33 | 34 | docs: 35 | MIX_ENV=docs mix docs 36 | 37 | publish: 38 | mix hex.publish 39 | mix hex.publish docs 40 | -------------------------------------------------------------------------------- /lib/eden/types.ex: -------------------------------------------------------------------------------- 1 | defmodule Eden.Character do 2 | defstruct char: nil 3 | 4 | def new(char), do: %Eden.Character{char: char} 5 | end 6 | 7 | defmodule Eden.Symbol do 8 | defstruct name: nil 9 | 10 | def new(name), do: %Eden.Symbol{name: name} 11 | end 12 | 13 | defmodule Eden.UUID do 14 | defstruct value: nil 15 | 16 | def new(value), do: %Eden.UUID{value: value} 17 | end 18 | 19 | defmodule Eden.Tag do 20 | defstruct name: nil, value: nil 21 | 22 | def new(name, value), do: %Eden.Tag{name: name, value: value} 23 | 24 | def inst(datetime) do 25 | Timex.parse!(datetime, "{RFC3339z}") 26 | end 27 | 28 | def uuid(value), do: %Eden.UUID{value: value} 29 | end 30 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Build 2 | on: 3 | push: 4 | branches: 5 | - '*' 6 | pull_request: 7 | branches: 8 | - master 9 | jobs: 10 | build: 11 | strategy: 12 | matrix: 13 | platform: [ubuntu-latest] 14 | runs-on: ${{ matrix.platform }} 15 | steps: 16 | - uses: actions/checkout@v2 17 | - name: Set up Elixir 18 | uses: actions/setup-elixir@v1 19 | with: 20 | elixir-version: '1.11' # Define the elixir version [required] 21 | otp-version: '22.3' # Define the OTP version [required] 22 | - name: Install Dependencies 23 | run: | 24 | mix local.rebar --force 25 | mix local.hex --force 26 | mix deps.get 27 | - run: mix deps.get 28 | - run: mix test 29 | -------------------------------------------------------------------------------- /config/config.exs: -------------------------------------------------------------------------------- 1 | # This file is responsible for configuring your application 2 | # and its dependencies with the aid of the Mix.Config module. 3 | use Mix.Config 4 | 5 | # This configuration is loaded before any dependency and is restricted 6 | # to this project. If another project depends on this project, this 7 | # file won't be loaded nor affect the parent project. For this reason, 8 | # if you want to provide default values for your application for third- 9 | # party users, it should be done in your mix.exs file. 10 | 11 | # Sample configuration: 12 | # 13 | # config :logger, :console, 14 | # level: :info, 15 | # format: "$date $time [$level] $metadata$message\n", 16 | # metadata: [:user_id] 17 | 18 | # It is also possible to import configuration files, relative to this 19 | # directory. For example, you can emulate configuration per environment 20 | # by uncommenting the line below and defining dev.exs, test.exs and such. 21 | # Configuration from the imported file will override the ones defined 22 | # here (which is why it is important to import them last). 23 | # 24 | # import_config "#{Mix.env}.exs" 25 | -------------------------------------------------------------------------------- /lib/eden/parser/node.ex: -------------------------------------------------------------------------------- 1 | defmodule Eden.Parser.Node do 2 | defstruct type: nil, location: nil, value: nil, children: [] 3 | @behaviour Access 4 | 5 | def fetch(map, key) do 6 | Map.fetch(map, key) 7 | end 8 | 9 | def get(map, key, value) do 10 | Map.get(map, key, value) 11 | end 12 | 13 | def pop(map, key) do 14 | Map.pop(map, key) 15 | end 16 | 17 | def get_and_update(%{} = map, key, fun) do 18 | Map.get_and_update(map, key, fun) 19 | end 20 | 21 | defimpl Inspect, for: __MODULE__ do 22 | import Inspect.Algebra 23 | 24 | def inspect(node, opts) do 25 | type_str = ":" <> Atom.to_string(node.type) 26 | value_str = if node.value, do: "\"" <> node.value <> "\" ", else: "" 27 | 28 | loc = node.location 29 | location_str = 30 | if loc do 31 | concat ["(", Integer.to_string(loc.line), ",", 32 | Integer.to_string(loc.col), ")"] 33 | else 34 | "" 35 | end 36 | 37 | level = Map.get(opts, :level, 0) 38 | opts = Map.put(opts, :level, level + 2) 39 | padding = String.duplicate(" ", level) 40 | 41 | concat [padding, "",type_str , " ", value_str, location_str, "\n"] 42 | ++ Enum.map(node.children, fn x -> to_doc(x, opts)end ) 43 | end 44 | end 45 | 46 | def reverse_children(node) do 47 | update_in(node, [:children], &Enum.reverse/1) 48 | end 49 | end 50 | -------------------------------------------------------------------------------- /mix.exs: -------------------------------------------------------------------------------- 1 | defmodule Eden.Mixfile do 2 | use Mix.Project 3 | 4 | @source_url "https://github.com/jfacorro/Eden/" 5 | @version "2.1.0" 6 | 7 | def project do 8 | [ 9 | app: :eden, 10 | version: @version, 11 | elixir: "~> 1.11", 12 | description: description(), 13 | package: package(), 14 | build_embedded: Mix.env() == :prod, 15 | start_permanent: Mix.env() == :prod, 16 | deps: deps(), 17 | docs: docs() 18 | ] 19 | end 20 | 21 | def application do 22 | [applications: [:timex, :elixir_array]] 23 | end 24 | 25 | defp deps do 26 | [ 27 | {:elixir_array, "~> 2.1.0"}, 28 | {:timex, "~> 3.1"}, 29 | {:exreloader, github: "jfacorro/exreloader", tag: "master", only: :dev}, 30 | {:ex_doc, "~> 0.23", only: :dev}, 31 | {:earmark, ">= 0.0.0", only: :dev} 32 | ] 33 | end 34 | 35 | defp description do 36 | """ 37 | edn (extensible data notation) encoder/decoder implemented in Elixir. 38 | """ 39 | end 40 | 41 | defp package do 42 | [ 43 | files: ["lib", "mix.exs", "README.md", "LICENSE", "CHANGELOG.md"], 44 | contributors: ["Juan Facorro"], 45 | licenses: ["Apache 2.0"], 46 | links: %{ 47 | "Changelog" => "#{@source_url}/blob/master/CHANGELOG.md", 48 | "GitHub" => @source_url, 49 | "edn format" => "https://github.com/edn-format/edn" 50 | } 51 | ] 52 | end 53 | 54 | defp docs do 55 | [ 56 | main: "readme", 57 | source_ref: @version, 58 | source_url: @source_url, 59 | extras: [ 60 | "README.md", 61 | "CHANGELOG.md" 62 | ] 63 | ] 64 | end 65 | end 66 | -------------------------------------------------------------------------------- /lib/eden/exception.ex: -------------------------------------------------------------------------------- 1 | defmodule Eden.Exception do 2 | 3 | defmodule Util do 4 | def token_message(token) do 5 | msg = "(#{inspect token.type}) #{inspect token.value}" 6 | if Map.has_key?(token, :location) do 7 | loc = token.location 8 | msg <> " at line #{inspect loc.line} and column #{inspect loc.col}." 9 | else 10 | msg 11 | end 12 | end 13 | end 14 | 15 | ## Lexer Exceptions 16 | 17 | defmodule UnexpectedInputError do 18 | defexception [:message] 19 | 20 | def exception(msg) do 21 | %UnexpectedInputError{message: msg} 22 | end 23 | end 24 | 25 | defmodule UnfinishedTokenError do 26 | defexception [:message] 27 | 28 | def exception(token) do 29 | %UnfinishedTokenError{message: Util.token_message(token)} 30 | end 31 | end 32 | 33 | ## Parser Exceptions 34 | 35 | defmodule UnexpectedTokenError do 36 | defexception [:message] 37 | def exception(token) do 38 | %UnexpectedTokenError{message: Util.token_message(token)} 39 | end 40 | end 41 | 42 | defmodule UnbalancedDelimiterError do 43 | defexception [:message] 44 | def exception(msg) do 45 | %UnbalancedDelimiterError{message: "#{inspect msg}"} 46 | end 47 | end 48 | 49 | defmodule OddExpressionCountError do 50 | defexception [:message] 51 | def exception(msg) do 52 | %OddExpressionCountError{message: "#{inspect msg}"} 53 | end 54 | end 55 | 56 | defmodule IncompleteTagError do 57 | defexception [:message] 58 | def exception(msg) do 59 | %IncompleteTagError{message: "#{inspect msg}"} 60 | end 61 | end 62 | 63 | defmodule MissingDiscardExpressionError do 64 | defexception [:message] 65 | def exception(msg) do 66 | %MissingDiscardExpressionError{message: "#{inspect msg}"} 67 | end 68 | end 69 | 70 | ## Decode Exceptions 71 | 72 | defmodule EmptyInputError do 73 | defexception [:message] 74 | def exception(msg) do 75 | %EmptyInputError{message: "#{inspect msg}"} 76 | end 77 | end 78 | 79 | defmodule NotImplementedError do 80 | defexception [:message] 81 | def exception(msg) when is_binary(msg) do 82 | %NotImplementedError{message: msg} 83 | end 84 | def exception({function, arity}) do 85 | function = Atom.to_string function 86 | %NotImplementedError{message: "#{function}/#{inspect arity}"} 87 | end 88 | end 89 | end 90 | -------------------------------------------------------------------------------- /lib/eden/encode.ex: -------------------------------------------------------------------------------- 1 | alias Eden.Encode 2 | alias Eden.Encode.Utils 3 | alias Eden.Character 4 | alias Eden.Symbol 5 | alias Eden.UUID 6 | alias Eden.Tag 7 | 8 | defprotocol Eden.Encode do 9 | @fallback_to_any true 10 | 11 | @spec encode(any) :: String.t 12 | def encode(value) 13 | end 14 | 15 | defmodule Eden.Encode.Utils do 16 | def wrap(str, first, last )do 17 | first <> str <> last 18 | end 19 | end 20 | 21 | defimpl Encode, for: Atom do 22 | def encode(atom) when atom in [nil, true, false] do 23 | Atom.to_string atom 24 | end 25 | def encode(atom) do 26 | ":" <> Atom.to_string atom 27 | end 28 | end 29 | 30 | defimpl Encode, for: Symbol do 31 | def encode(symbol) do symbol.name end 32 | end 33 | 34 | defimpl Encode, for: BitString do 35 | def encode(string) do "\"#{string}\"" end 36 | end 37 | 38 | defimpl Encode, for: Character do 39 | def encode(char) do "\\#{char.char}" end 40 | end 41 | 42 | defimpl Encode, for: Integer do 43 | def encode(int) do "#{inspect int}" end 44 | end 45 | 46 | defimpl Encode, for: Float do 47 | def encode(float) do "#{inspect float}" end 48 | end 49 | 50 | defimpl Encode, for: List do 51 | def encode(list) do 52 | list 53 | |> Enum.map(&Encode.encode/1) 54 | |> Enum.join(", ") 55 | |> Utils.wrap("(", ")") 56 | end 57 | end 58 | 59 | defimpl Encode, for: Array do 60 | def encode(array) do 61 | array 62 | |> Array.to_list 63 | |> Enum.map(&Encode.encode/1) 64 | |> Enum.join(", ") 65 | |> Utils.wrap("[", "]") 66 | end 67 | end 68 | 69 | defimpl Encode, for: Map do 70 | def encode(map) do 71 | map 72 | |> Map.to_list 73 | |> Enum.map(fn {k, v} -> Encode.encode(k) <> " " <> Encode.encode(v) end) 74 | |> Enum.join(", ") 75 | |> Utils.wrap("{", "}") 76 | end 77 | end 78 | 79 | defimpl Encode, for: MapSet do 80 | def encode(set) do 81 | set 82 | |> Enum.map(&Encode.encode/1) 83 | |> Enum.join(", ") 84 | |> Utils.wrap("#\{", "}") 85 | end 86 | end 87 | 88 | defimpl Encode, for: Tag do 89 | def encode(tag) do 90 | value = Encode.encode(tag.value) 91 | "##{tag.name} #{value}" 92 | end 93 | end 94 | 95 | defimpl Encode, for: UUID do 96 | def encode(uuid) do 97 | Encode.encode(Tag.new("uuid", uuid.value)) 98 | end 99 | end 100 | 101 | defimpl Encode, for: DateTime do 102 | def encode(datetime) do 103 | value = Timex.format!(datetime, "{RFC3339z}") 104 | Encode.encode(Tag.new("inst", value)) 105 | end 106 | end 107 | 108 | defimpl Encode, for: Any do 109 | def encode(struct) when is_map(struct) do 110 | Encode.encode(Map.from_struct(struct)) 111 | end 112 | def encode(value) do 113 | raise %Protocol.UndefinedError{protocol: Encode, value: value} 114 | end 115 | end 116 | -------------------------------------------------------------------------------- /lib/eden/decode.ex: -------------------------------------------------------------------------------- 1 | defmodule Eden.Decode do 2 | alias Eden.Parser.Node 3 | alias Eden.Character 4 | alias Eden.Symbol 5 | alias Eden.Tag 6 | alias Eden.Exception, as: Ex 7 | require Integer 8 | 9 | def decode(children, opts) when is_list(children) do 10 | Enum.map(children, fn x -> decode(x, opts) end) 11 | end 12 | def decode(%Node{type: :root, children: children}, opts) do 13 | decode(children, opts) 14 | end 15 | def decode(%Node{type: :nil}, _opts) do 16 | :nil 17 | end 18 | def decode(%Node{type: :true}, _opts) do 19 | :true 20 | end 21 | def decode(%Node{type: :false}, _opts) do 22 | :false 23 | end 24 | def decode(%Node{type: :string, value: value}, _opts) do 25 | value 26 | end 27 | def decode(%Node{type: :character, value: value}, _opts) do 28 | %Character{char: value} 29 | end 30 | def decode(%Node{type: :symbol, value: value}, _opts) do 31 | %Symbol{name: value} 32 | end 33 | def decode(%Node{type: :keyword, value: value}, _opts) do 34 | String.to_atom(value) 35 | end 36 | def decode(%Node{type: :integer, value: value}, _opts) do 37 | value = String.trim_trailing(value, "N") 38 | :erlang.binary_to_integer(value) 39 | end 40 | def decode(%Node{type: :float, value: value}, _opts) do 41 | value = String.trim_trailing(value, "M") 42 | # Elixir/Erlang don't convert to float if there 43 | # is no decimal part. 44 | final_value = if not String.contains?(value, ".") do 45 | if String.match?(value, ~r/[eE]/) do 46 | String.replace(value, ~r/[eE]/, ".0E") 47 | else 48 | value <> ".0" 49 | end 50 | else 51 | value 52 | end 53 | :erlang.binary_to_float(final_value) 54 | end 55 | def decode(%Node{type: :list, children: children}, opts) do 56 | decode(children, opts) 57 | end 58 | def decode(%Node{type: :vector, children: children}, opts) do 59 | children 60 | |> decode(opts) 61 | |> Array.from_list 62 | end 63 | def decode(%Node{type: :map, children: children} = node, opts) do 64 | if Integer.is_odd(length children) do 65 | raise Ex.OddExpressionCountError, node 66 | end 67 | children 68 | |> decode(opts) 69 | |> Enum.chunk_every(2) 70 | |> Enum.map(fn [a, b] -> {a, b} end) 71 | |> Enum.into(%{}) 72 | end 73 | def decode(%Node{type: :set, children: children}, opts) do 74 | children 75 | |> decode(opts) 76 | |> Enum.into(MapSet.new) 77 | end 78 | def decode(%Node{type: :tag, value: name, children: [child]}, opts) do 79 | case Map.get(opts[:handlers], name) do 80 | nil -> 81 | %Tag{name: name, value: decode(child, opts)} 82 | handler -> 83 | handler.(decode(child, opts)) 84 | end 85 | end 86 | def decode(%Node{type: type}, _opts) do 87 | raise "Unrecognized node type: #{inspect type}" 88 | end 89 | end 90 | -------------------------------------------------------------------------------- /lib/eden.ex: -------------------------------------------------------------------------------- 1 | defmodule Eden do 2 | import Eden.Parser 3 | 4 | @moduledoc """ 5 | Provides functions to `encode/1` and `decode/2` between *Elixir* and 6 | *edn* data format. 7 | """ 8 | 9 | alias Eden.Encode 10 | alias Eden.Decode 11 | alias Eden.Exception, as: Ex 12 | 13 | @default_handlers %{"inst" => &Eden.Tag.inst/1, 14 | "uuid" => &Eden.Tag.uuid/1} 15 | 16 | @doc """ 17 | Encodes an *Elixir* term that implements the `Eden.Encode` protocol. 18 | When the term is a nested data structure (e.g. `List`, `Map`, etc.), 19 | all children should also implement `Eden.Encode` protocol for the 20 | encoding to be successful. 21 | 22 | There is an implementation for the most common *Elixir* data types: 23 | 24 | - `Atom` 25 | - `BitString` (binary) 26 | - `Integer` 27 | - `Float` 28 | - `Map` 29 | - `List` 30 | - `MapSet` 31 | 32 | There are also implementations for the following custom *Elixir* data 33 | types in order to support native *edn* types: 34 | 35 | - `Eden.Symbol` 36 | - `Eden.Character` 37 | - `Array` (vector) 38 | - `Eden.Tag` (tagged value) 39 | 40 | Since the *edn* specification requires every implementation to 41 | provide handlers for tags `uuid` and `inst`, the following data 42 | types also have an implementation for `Eden.Encode`: 43 | 44 | - `Eden.UUID` (`#uuid`) 45 | - `Timex.DatetTime` (`#inst`) 46 | 47 | ## Examples 48 | 49 | iex> Eden.encode([1, 2]) 50 | {:ok, "(1, 2)"} 51 | 52 | iex> Eden.encode(%{a: 1, b: 2, c: 3}) 53 | {:ok, "{:a 1, :b 2, :c 3}"} 54 | 55 | iex> Eden.encode({:a, 1}) 56 | {:error, Protocol.UndefinedError} 57 | """ 58 | @spec encode(Encode.t) :: {:ok, String.t} | {:error, atom} 59 | def encode(data) do 60 | try do 61 | {:ok, encode!(data)} 62 | rescue 63 | e -> {:error, e.__struct__} 64 | end 65 | end 66 | 67 | @doc """ 68 | Same as `encode/1` but raises an error if the term could not 69 | be encoded. 70 | 71 | Returns the function result otherwise. 72 | """ 73 | @spec encode!(Encode.t) :: String.t 74 | def encode!(data) do 75 | Encode.encode(data) 76 | end 77 | 78 | @doc """ 79 | Decodes a string containing *edn* data into *Elixir* data 80 | structures. For a detailed list on the mapping between 81 | *edn* and *Elixir* check the documentation in the project's 82 | [page](https://github.com/jfacorro/Eden). 83 | 84 | When the string contains a single expression it is decoded 85 | and returned. Otherwise, if there are multiple expressions, 86 | then a list with all parsed expressions is returned. 87 | 88 | ## Examples 89 | 90 | iex> Eden.decode("{:a 1 :b 2}") 91 | {:ok, %{a: 1, b: 2}} 92 | 93 | iex> Eden.decode("(hello :world \\!)") 94 | {:ok, [%Eden.Symbol{name: "hello"}, :world, %Eden.Character{char: "!"}] 95 | 96 | iex> Eden.decode("[1 2 3 4]") 97 | {:ok, #Array<[1, 2, 3, 4], fixed=false, default=nil>} 98 | 99 | iex> Eden.decode("nil true false") 100 | {:ok, #Array<[1, 2, 3, 4], fixed=false, default=nil>} 101 | 102 | iex> Eden.decode("nil true false .") 103 | {:error, Eden.Exception.UnexpectedInputError} 104 | """ 105 | @spec decode(String.t, Keyword.t) :: {:ok, any} | {:error, atom} 106 | def decode(input, opts \\ []) do 107 | try do 108 | {:ok, decode!(input, opts)} 109 | rescue 110 | e -> {:error, e.__struct__} 111 | end 112 | end 113 | 114 | @doc """ 115 | Same as `decode/1` but raises an error if the term could not 116 | be encoded. 117 | 118 | Returns the function result otherwise. 119 | """ 120 | @spec decode!(String.t, Keyword.t) :: any 121 | def decode!(input, opts \\ []) do 122 | tree = parse(input, location: true) 123 | handlers = Map.merge(@default_handlers, opts[:handlers] || %{}) 124 | opts = [handlers: handlers] 125 | case Decode.decode(tree, opts) do 126 | [] -> raise Ex.EmptyInputError, input 127 | [data] -> data 128 | data -> data 129 | end 130 | end 131 | end 132 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Eden 2 | ===== 3 | 4 | [![GitHub](https://github.com/jfacorro/Eden/workflows/Build/badge.svg)](https://github.com/jfacorro/Eden/actions?query=workflow%3ABuild) 5 | [![Hex.pm](https://img.shields.io/hexpm/v/eden.svg?style=flat-square)](https://hex.pm/packages/eden) 6 | [![Hexdocs.pm](https://img.shields.io/badge/hex-docs-lightgreen.svg)](https://hexdocs.pm/eden/) 7 | [![Hex.pm](https://img.shields.io/hexpm/dt/eden.svg?style=flat-square)](https://hex.pm/packages/eden) 8 | [![Hex.pm](https://img.shields.io/hexpm/l/eden.svg?style=flat-square)](https://hex.pm/packages/eden) 9 | [![GitHub](https://img.shields.io/github/last-commit/jfacorro/Eden.svg)](https://github.com/jfacorro/Eden/commits/master) 10 | 11 | [edn](https://github.com/edn-format/edn) (extensible data notation) encoder/decoder implemented in Elixir. 12 | 13 | ## Usage 14 | 15 | Include Eden as a dependency in your Elixir project by adding it in your `deps` list: 16 | 17 | ```elixir 18 | def deps do 19 | [{:eden, "~> 0.1.2"}] 20 | end 21 | ``` 22 | 23 | Eden is a library application and as such doesn't specify an application callback module. Even so, if you would like to build a release that includes Eden, you need to add it as an application dependency in your `mix.exs`: 24 | 25 | ```elixir 26 | def application do 27 | [applications: [:eden]] 28 | end 29 | ``` 30 | 31 | ## Examples 32 | 33 | ```elixir 34 | iex> Eden.encode([1, 2]) 35 | {:ok, "(1, 2)"} 36 | 37 | iex> Eden.encode(%{a: 1, b: 2, c: 3}) 38 | {:ok, "{:a 1, :b 2, :c 3}"} 39 | 40 | iex> Eden.encode({:a, 1}) 41 | {:error, Protocol.UndefinedError} 42 | 43 | iex> Eden.decode("{:a 1 :b 2}") 44 | {:ok, %{a: 1, b: 2}} 45 | 46 | iex> Eden.decode("(hello :world \\!)") 47 | {:ok, [%Eden.Symbol{name: "hello"}, :world, %Eden.Character{char: "!"}] 48 | 49 | iex> Eden.decode("[1 2 3 4]") 50 | {:ok, #Array<[1, 2, 3, 4], fixed=false, default=nil>} 51 | 52 | iex> Eden.decode("nil true false") 53 | {:ok, [nil, true, false]} 54 | 55 | iex> Eden.decode("nil true false .") 56 | {:error, Eden.Exception.UnexpectedInputError} 57 | ``` 58 | 59 | ## Data Structures Mapping: **edn** <-> **Elixir** 60 | 61 | | Edn | Elixir | 62 | |---|---| 63 | | `nil` | `:nil = nil` | 64 | | `true` | `:true = true` | 65 | | `false` | `:false = false` | 66 | | `string` | `String` | 67 | | `character` | `Eden.Character` | 68 | | `symbol` | `Eden.Symbol` | 69 | | `keyword` | `Atom` | 70 | | `integer` | `Integer` | 71 | | `float` | `Float` | 72 | | `list` | `List` | 73 | | `vector` | `Array` | 74 | | `map` | `Map` | 75 | | `set` | `MapSet` | 76 | | `#inst` | `Timex.DateTime` | 77 | | `#uuid` | `Eden.UUID` | 78 | 79 | ## Further Considerations 80 | 81 | ### `Character` 82 | 83 | There is no way of distinguishing a common integer from the representation of a character in a `String` or in `Char lists`. This forces the creation of a new representation for this type so it can be correctly translated from and to **edn**. 84 | 85 | ### Arbitrary Precision `Integer` and `Float` 86 | 87 | The Erlang VM (EVM) only provides arbitrary precision integers so all integers will be of this type this and the `N` modifier will be ignored when parsing an **edn** integer. 88 | 89 | On the other hand native arbitrary precision floating point numbers are not provided by the EVM so all values of type `float` will be represented according to what the EVM supports. 90 | 91 | ### `Keyword` and `Symbol` Representation 92 | 93 | On one hand the decision to translate **edn** `keyword`s as Elixir `atom`s comes from the fact these two data types are given a similar usage on both languages. On the other, it might get awkward really fast using a new `Eden.Symbol` struct as the representation for **edn**'s `symbol`s so this might change. 94 | 95 | ### `vector` 96 | 97 | There is no constant lookup or nearly constant indexed data structure like **edn**'s `vector` other than the `:array` data structure implemented in one of Erlang's standard library modules. Until there is a better implementation for this `Eden` will use [`Array`](https://github.com/takscape/elixir-array), an Elixir wrapper library for Erlang's array. 98 | 99 | ## **edn** grammar 100 | 101 | ``` 102 | expr -> literal | map | list | vector | tagged_value 103 | 104 | literal -> nil | true | false | keyword | symbol | integer | float | string 105 | 106 | map -> { pair* } 107 | pair -> expr expr 108 | 109 | list -> ( expr* ) 110 | 111 | vector -> [ expr* ] 112 | 113 | tagged_value -> tag expr 114 | ``` 115 | -------------------------------------------------------------------------------- /test/eden_test.exs: -------------------------------------------------------------------------------- 1 | defmodule EdenTest do 2 | use ExUnit.Case 3 | import Eden 4 | alias Eden.Character 5 | alias Eden.Symbol 6 | alias Eden.UUID 7 | alias Eden.Tag 8 | alias Eden.Exception, as: Ex 9 | 10 | ## Decode 11 | 12 | test "Decode Empty Input" do 13 | e = %Ex.EmptyInputError{} 14 | assert decode("") == {:error, e.__struct__} 15 | 16 | assert_raise Ex.EmptyInputError, fn -> 17 | decode!("") 18 | end 19 | 20 | end 21 | 22 | test "Decode Literals" do 23 | assert decode!("nil") == nil 24 | assert decode!("true") == true 25 | assert decode!("false") == false 26 | assert decode!("false false") == [false, false] 27 | 28 | assert decode!("\"hello world!\"") == "hello world!" 29 | assert decode!("\"hello \\n world!\"") == "hello \n world!" 30 | 31 | assert decode!("\\n") == %Character{char: "n"} 32 | assert decode!("\\z") == %Character{char: "z"} 33 | 34 | assert decode!("a-symbol") == %Symbol{name: "a-symbol"} 35 | assert decode!(":the-keyword") == :'the-keyword' 36 | 37 | assert decode!("42") == 42 38 | assert decode!("42N") == 42 39 | 40 | assert decode!("42.0") == 42.0 41 | assert decode!("42M") == 42.0 42 | assert decode!("42.0e3") == 42000.0 43 | assert decode!("42e-3") == 0.042 44 | assert decode!("42E-1") == 4.2 45 | assert decode!("42.01E+1") == 420.1 46 | end 47 | 48 | test "Decode List" do 49 | assert decode!("(1 :a 42.0)") == [1, :a, 42.0] 50 | end 51 | 52 | test "Decode Vector" do 53 | array = Array.from_list([1, :a, 42.0]) 54 | assert decode!("[1 :a 42.0]") == array 55 | end 56 | 57 | test "Decode Map" do 58 | map = %{name: "John", age: 42} 59 | assert decode!("{:name \"John\" :age 42}") == map 60 | 61 | assert_raise Ex.OddExpressionCountError, fn -> 62 | decode!("{:name \"John\" :age}") 63 | end 64 | end 65 | 66 | test "Decode Set" do 67 | set = Enum.into([:name, "John", :age, 42], MapSet.new) 68 | assert decode!("#\{:name \"John\" :age 42}") == set 69 | end 70 | 71 | test "Decode Tag" do 72 | date = Timex.parse!("1985-04-12T23:20:50.52Z", "{RFC3339z}") 73 | assert decode!("#inst \"1985-04-12T23:20:50.52Z\"") == date 74 | assert decode!("#uuid \"f81d4fae-7dec-11d0-a765-00a0c91e6bf6\"") == %UUID{value: "f81d4fae-7dec-11d0-a765-00a0c91e6bf6"} 75 | 76 | assert decode!("#custom/tag (1 2 3)") == %Tag{name: "custom/tag", value: [1, 2, 3]} 77 | handlers = %{"custom/tag" => &custom_tag_handler/1} 78 | assert decode!("#custom/tag (1 2 3)", handlers: handlers) == [:a, :b, :c] 79 | end 80 | 81 | ## Encode 82 | 83 | test "Encode Literals" do 84 | assert encode!(nil) == "nil" 85 | assert encode!(true) == "true" 86 | assert encode!(false) == "false" 87 | 88 | assert encode!("hello world!") == "\"hello world!\"" 89 | assert encode!("hello \n world!") == "\"hello \n world!\"" 90 | 91 | assert encode!(Character.new("n")) == "\\n" 92 | assert encode!(Character.new("z")) == "\\z" 93 | 94 | assert encode!(Symbol.new("a-symbol")) == "a-symbol" 95 | assert encode!(:"the-keyword") == ":the-keyword" 96 | 97 | assert encode!(42) == "42" 98 | 99 | assert encode!(42.0) == "42.0" 100 | assert encode!(42.0e3) == "4.2e4" 101 | assert encode!(42.0e-3) == "0.042" 102 | assert encode!(42.0e-1) == "4.2" 103 | assert encode!(42.01E+1) == "420.1" 104 | end 105 | 106 | test "Encode List" do 107 | assert encode!([1, :a, 42.0]) == "(1, :a, 42.0)" 108 | end 109 | 110 | test "Encode Vector" do 111 | array = Array.from_list([1, :a, 42.0]) 112 | assert encode!(array) == "[1, :a, 42.0]" 113 | end 114 | 115 | test "Encode Map" do 116 | map = %{name: "John", age: 42} 117 | assert encode!(map) == "{:age 42, :name \"John\"}" 118 | end 119 | 120 | test "Encode Set" do 121 | set = Enum.into([:name, "John", :age, 42], MapSet.new) 122 | assert encode!(set) == "#\{42, :age, :name, \"John\"}" 123 | end 124 | 125 | test "Encode Tag" do 126 | date = Timex.parse!("1985-04-12T23:20:50.52Z", "{RFC3339z}") 127 | assert encode!(date) == "#inst \"1985-04-12T23:20:50.52Z\"" 128 | uuid = UUID.new("f81d4fae-7dec-11d0-a765-00a0c91e6bf6") 129 | assert encode!(uuid) == "#uuid \"f81d4fae-7dec-11d0-a765-00a0c91e6bf6\"" 130 | 131 | some_tag = Tag.new("custom/tag", :joni) 132 | assert encode!(some_tag) == "#custom/tag :joni" 133 | end 134 | 135 | test "Encode Fallback to Any" do 136 | node = %Eden.Parser.Node{} 137 | map = Map.from_struct(node) 138 | assert encode!(node) == encode!(map) 139 | end 140 | 141 | test "Encode Unknown Type" do 142 | e = %Protocol.UndefinedError{} 143 | assert encode(self()) == {:error, e.__struct__} 144 | 145 | assert_raise Protocol.UndefinedError, fn -> 146 | encode!(self()) 147 | end 148 | 149 | try do 150 | encode!(self()) 151 | rescue 152 | e in Protocol.UndefinedError -> 153 | assert e.protocol == Eden.Encode 154 | assert e.value == self() 155 | end 156 | 157 | end 158 | 159 | defp custom_tag_handler(value) when is_list(value) do 160 | mapping = %{1 => :a, 2 => :b, 3 => :c} 161 | Enum.map(value, fn x -> mapping[x] end) 162 | end 163 | end 164 | -------------------------------------------------------------------------------- /mix.lock: -------------------------------------------------------------------------------- 1 | %{ 2 | "certifi": {:hex, :certifi, "2.5.2", "b7cfeae9d2ed395695dd8201c57a2d019c0c43ecaf8b8bcb9320b40d6662f340", [:rebar3], [{:parse_trans, "~>3.3", [hex: :parse_trans, repo: "hexpm", optional: false]}], "hexpm", "3b3b5f36493004ac3455966991eaf6e768ce9884693d9968055aeeeb1e575040"}, 3 | "combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"}, 4 | "earmark": {:hex, :earmark, "1.4.10", "bddce5e8ea37712a5bfb01541be8ba57d3b171d3fa4f80a0be9bcf1db417bcaf", [:mix], [{:earmark_parser, ">= 1.4.10", [hex: :earmark_parser, repo: "hexpm", optional: false]}], "hexpm", "12dbfa80810478e521d3ffb941ad9fbfcbbd7debe94e1341b4c4a1b2411c1c27"}, 5 | "earmark_parser": {:hex, :earmark_parser, "1.4.10", "6603d7a603b9c18d3d20db69921527f82ef09990885ed7525003c7fe7dc86c56", [:mix], [], "hexpm", "8e2d5370b732385db2c9b22215c3f59c84ac7dda7ed7e544d7c459496ae519c0"}, 6 | "elixir_array": {:hex, :elixir_array, "2.1.0", "580293f82afdd63be880d69d3b1cc829fe6994f4b1442ee809ccb7199ee7fa13", [:mix], [], "hexpm", "9425e43cf7df7eaea6ac03e25e317835b52212ec374d71029a8fd02c1c32648c"}, 7 | "ex_doc": {:hex, :ex_doc, "0.23.0", "a069bc9b0bf8efe323ecde8c0d62afc13d308b1fa3d228b65bca5cf8703a529d", [:mix], [{:earmark_parser, "~> 1.4.0", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm", "f5e2c4702468b2fd11b10d39416ddadd2fcdd173ba2a0285ebd92c39827a5a16"}, 8 | "exreloader": {:git, "https://github.com/jfacorro/exreloader.git", "c54f341284597d9efbeb0fe05001ff442a87074c", [tag: "master"]}, 9 | "gettext": {:hex, :gettext, "0.18.2", "7df3ea191bb56c0309c00a783334b288d08a879f53a7014341284635850a6e55", [:mix], [], "hexpm", "f9f537b13d4fdd30f3039d33cb80144c3aa1f8d9698e47d7bcbcc8df93b1f5c5"}, 10 | "hackney": {:hex, :hackney, "1.16.0", "5096ac8e823e3a441477b2d187e30dd3fff1a82991a806b2003845ce72ce2d84", [:rebar3], [{:certifi, "2.5.2", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.1", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.3.0", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.6", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "3bf0bebbd5d3092a3543b783bf065165fa5d3ad4b899b836810e513064134e18"}, 11 | "idna": {:hex, :idna, "6.0.1", "1d038fb2e7668ce41fbf681d2c45902e52b3cb9e9c77b55334353b222c2ee50c", [:rebar3], [{:unicode_util_compat, "0.5.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "a02c8a1c4fd601215bb0b0324c8a6986749f807ce35f25449ec9e69758708122"}, 12 | "makeup": {:hex, :makeup, "1.0.5", "d5a830bc42c9800ce07dd97fa94669dfb93d3bf5fcf6ea7a0c67b2e0e4a7f26c", [:mix], [{:nimble_parsec, "~> 0.5 or ~> 1.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cfa158c02d3f5c0c665d0af11512fed3fba0144cf1aadee0f2ce17747fba2ca9"}, 13 | "makeup_elixir": {:hex, :makeup_elixir, "0.15.0", "98312c9f0d3730fde4049985a1105da5155bfe5c11e47bdc7406d88e01e4219b", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.1", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "75ffa34ab1056b7e24844c90bfc62aaf6f3a37a15faa76b07bc5eba27e4a8b4a"}, 14 | "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"}, 15 | "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"}, 16 | "nimble_parsec": {:hex, :nimble_parsec, "1.1.0", "3a6fca1550363552e54c216debb6a9e95bd8d32348938e13de5eda962c0d7f89", [:mix], [], "hexpm", "08eb32d66b706e913ff748f11694b17981c0b04a33ef470e33e11b3d3ac8f54b"}, 17 | "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm", "17ef63abde837ad30680ea7f857dd9e7ced9476cdd7b0394432af4bfc241b960"}, 18 | "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.6", "cf344f5692c82d2cd7554f5ec8fd961548d4fd09e7d22f5b62482e5aeaebd4b0", [:make, :mix, :rebar3], [], "hexpm", "bdb0d2471f453c88ff3908e7686f86f9be327d065cc1ec16fa4540197ea04680"}, 19 | "timex": {:hex, :timex, "3.6.2", "845cdeb6119e2fef10751c0b247b6c59d86d78554c83f78db612e3290f819bc2", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "26030b46199d02a590be61c2394b37ea25a3664c02fafbeca0b24c972025d47a"}, 20 | "tzdata": {:hex, :tzdata, "1.0.4", "a3baa4709ea8dba552dca165af6ae97c624a2d6ac14bd265165eaa8e8af94af6", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "b02637db3df1fd66dd2d3c4f194a81633d0e4b44308d36c1b2fdfd1e4e6f169b"}, 21 | "unicode_util_compat": {:hex, :unicode_util_compat, "0.5.0", "8516502659002cec19e244ebd90d312183064be95025a319a6c7e89f4bccd65b", [:rebar3], [], "hexpm", "d48d002e15f5cc105a696cf2f1bbb3fc72b4b770a184d8420c8db20da2674b38"}, 22 | } 23 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## [2.1.0](https://github.com/jfacorro/Eden/tree/2.1.0) (2020-10-31) 4 | 5 | [Full Changelog](https://github.com/jfacorro/Eden/compare/2.0.0...2.1.0) 6 | 7 | **Merged pull requests:** 8 | 9 | - \[\#39\] Adapt library for publishing [\#42](https://github.com/jfacorro/Eden/pull/42) ([jfacorro](https://github.com/jfacorro)) 10 | - \[\#39\] Update CHANGELOG [\#41](https://github.com/jfacorro/Eden/pull/41) ([jfacorro](https://github.com/jfacorro)) 11 | - Update Travis CI configuration with latest OTP and Elixir versions [\#40](https://github.com/jfacorro/Eden/pull/40) ([jfacorro](https://github.com/jfacorro)) 12 | - Bumps elixir [\#38](https://github.com/jfacorro/Eden/pull/38) ([naomijub](https://github.com/naomijub)) 13 | 14 | ## [2.0.0](https://github.com/jfacorro/Eden/tree/2.0.0) (2018-03-04) 15 | 16 | [Full Changelog](https://github.com/jfacorro/Eden/compare/1.0.2...2.0.0) 17 | 18 | **Merged pull requests:** 19 | 20 | - Migrated to Elixir 1.5 and Timex 3.1 [\#37](https://github.com/jfacorro/Eden/pull/37) ([f34nk](https://github.com/f34nk)) 21 | 22 | ## [1.0.2](https://github.com/jfacorro/Eden/tree/1.0.2) (2017-06-14) 23 | 24 | [Full Changelog](https://github.com/jfacorro/Eden/compare/1.0.1...1.0.2) 25 | 26 | **Fixed bugs:** 27 | 28 | - The input ":" is decoded as an Elixir atom instead of generating an error [\#22](https://github.com/jfacorro/Eden/issues/22) 29 | 30 | **Merged pull requests:** 31 | 32 | - \[Fixes \#22\] Throw exception when trying to parse a single colon [\#36](https://github.com/jfacorro/Eden/pull/36) ([jfacorro](https://github.com/jfacorro)) 33 | 34 | ## [1.0.1](https://github.com/jfacorro/Eden/tree/1.0.1) (2017-06-14) 35 | 36 | [Full Changelog](https://github.com/jfacorro/Eden/compare/1.0.0...1.0.1) 37 | 38 | **Closed issues:** 39 | 40 | - Could not parse file [\#32](https://github.com/jfacorro/Eden/issues/32) 41 | 42 | **Merged pull requests:** 43 | 44 | - \[\#32\] Specify utf8 encoding for all binaries built in lexer [\#35](https://github.com/jfacorro/Eden/pull/35) ([jfacorro](https://github.com/jfacorro)) 45 | - \[Fixes \#32\] Specify utf8 enconding when buliding single char binaries in lexer [\#33](https://github.com/jfacorro/Eden/pull/33) ([jfacorro](https://github.com/jfacorro)) 46 | 47 | ## [1.0.0](https://github.com/jfacorro/Eden/tree/1.0.0) (2016-09-16) 48 | 49 | [Full Changelog](https://github.com/jfacorro/Eden/compare/0.1.3...1.0.0) 50 | 51 | **Implemented enhancements:** 52 | 53 | - Doesn't compile for Elixir 1.3.2 [\#30](https://github.com/jfacorro/Eden/issues/30) 54 | 55 | **Merged pull requests:** 56 | 57 | - Force build [\#28](https://github.com/jfacorro/Eden/pull/28) ([jfacorro](https://github.com/jfacorro)) 58 | 59 | ## [0.1.3](https://github.com/jfacorro/Eden/tree/0.1.3) (2015-06-13) 60 | 61 | [Full Changelog](https://github.com/jfacorro/Eden/compare/0.1.2...0.1.3) 62 | 63 | **Implemented enhancements:** 64 | 65 | - Version bump 0.1.2 [\#18](https://github.com/jfacorro/Eden/issues/18) 66 | 67 | **Closed issues:** 68 | 69 | - Fix example in README.md and correct the Erlang OTP [\#25](https://github.com/jfacorro/Eden/issues/25) 70 | - Upload generated docs to gh-pages branch [\#23](https://github.com/jfacorro/Eden/issues/23) 71 | - Rename project ExEdn to Eden [\#21](https://github.com/jfacorro/Eden/issues/21) 72 | - Publish in Hex [\#17](https://github.com/jfacorro/Eden/issues/17) 73 | 74 | **Merged pull requests:** 75 | 76 | - Bump version [\#34](https://github.com/jfacorro/Eden/pull/34) ([jfacorro](https://github.com/jfacorro)) 77 | - \[Closes \#30\] Fix warning and replaced Access protocol for behaviour [\#31](https://github.com/jfacorro/Eden/pull/31) ([jfacorro](https://github.com/jfacorro)) 78 | - Version bump to 0.1.3 [\#27](https://github.com/jfacorro/Eden/pull/27) ([jfacorro](https://github.com/jfacorro)) 79 | - \[Closes \#25\] Fix README add application deps [\#26](https://github.com/jfacorro/Eden/pull/26) ([jfacorro](https://github.com/jfacorro)) 80 | - \[\#21\] Rename from ExEdn to Eden [\#24](https://github.com/jfacorro/Eden/pull/24) ([jfacorro](https://github.com/jfacorro)) 81 | - \[Closes \#17\] Publish in hex.pm [\#20](https://github.com/jfacorro/Eden/pull/20) ([jfacorro](https://github.com/jfacorro)) 82 | 83 | ## [0.1.2](https://github.com/jfacorro/Eden/tree/0.1.2) (2015-06-13) 84 | 85 | [Full Changelog](https://github.com/jfacorro/Eden/compare/0.1.1...0.1.2) 86 | 87 | **Closed issues:** 88 | 89 | - More information for the Protocol.UndefinedError raised by ExEdn.Encode [\#14](https://github.com/jfacorro/Eden/issues/14) 90 | - Docs and specs for almost everything [\#10](https://github.com/jfacorro/Eden/issues/10) 91 | 92 | **Merged pull requests:** 93 | 94 | - \[\#18\] Version bump 0.1.2 [\#19](https://github.com/jfacorro/Eden/pull/19) ([jfacorro](https://github.com/jfacorro)) 95 | - \[Closes \#10\] Docs and specs [\#16](https://github.com/jfacorro/Eden/pull/16) ([jfacorro](https://github.com/jfacorro)) 96 | - \[\#14\] Added more information to Protocol.UndefinedError [\#15](https://github.com/jfacorro/Eden/pull/15) ([jfacorro](https://github.com/jfacorro)) 97 | 98 | ## [0.1.1](https://github.com/jfacorro/Eden/tree/0.1.1) (2015-06-10) 99 | 100 | [Full Changelog](https://github.com/jfacorro/Eden/compare/0.1.0...0.1.1) 101 | 102 | **Closed issues:** 103 | 104 | - Add an Encode protocol implementation for Any [\#12](https://github.com/jfacorro/Eden/issues/12) 105 | 106 | **Merged pull requests:** 107 | 108 | - \[Closes \#12\] Encode protocol implementation for Any, but actually only for s… [\#13](https://github.com/jfacorro/Eden/pull/13) ([jfacorro](https://github.com/jfacorro)) 109 | 110 | ## [0.1.0](https://github.com/jfacorro/Eden/tree/0.1.0) (2015-06-10) 111 | 112 | [Full Changelog](https://github.com/jfacorro/Eden/compare/63dbab0a9c19bb73a56108dfcda7b72f06865fd4...0.1.0) 113 | 114 | **Closed issues:** 115 | 116 | - Elixir -\> edn [\#8](https://github.com/jfacorro/Eden/issues/8) 117 | - edn \(parse tree\) -\> Elixir [\#5](https://github.com/jfacorro/Eden/issues/5) 118 | - Add line and column information to lexer [\#4](https://github.com/jfacorro/Eden/issues/4) 119 | - Parser [\#2](https://github.com/jfacorro/Eden/issues/2) 120 | - Lexer [\#1](https://github.com/jfacorro/Eden/issues/1) 121 | 122 | **Merged pull requests:** 123 | 124 | - \[Closes \#8\] Elixir -\> edn [\#11](https://github.com/jfacorro/Eden/pull/11) ([jfacorro](https://github.com/jfacorro)) 125 | - \[Closes \#5\] Elixir from parse tree [\#9](https://github.com/jfacorro/Eden/pull/9) ([jfacorro](https://github.com/jfacorro)) 126 | - \[\#1\] Lexer [\#3](https://github.com/jfacorro/Eden/pull/3) ([jfacorro](https://github.com/jfacorro)) 127 | 128 | 129 | 130 | \* *This Changelog was automatically generated by [github_changelog_generator](https://github.com/github-changelog-generator/github-changelog-generator)* 131 | -------------------------------------------------------------------------------- /test/eden/parser_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Eden.ParserTest do 2 | use ExUnit.Case 3 | import Eden.Parser 4 | alias Eden.Parser 5 | alias Eden.Exception, as: Ex 6 | 7 | test "Whitespace" do 8 | assert parse(",,, ") == node(:root, nil, []) 9 | assert parse(" \n \t, \r") == node(:root, nil, []) 10 | end 11 | 12 | test "Literals" do 13 | root = node(:root, nil, [node(:nil, "nil"), 14 | node(:true, "true"), 15 | node(:false, "false")]) 16 | assert parse("nil true false") == root 17 | 18 | root = node(:root, nil, [node(:integer, "1")]) 19 | assert parse("1") == root 20 | root = node(:root, nil, [node(:integer, "29382329N")]) 21 | assert parse("29382329N") == root 22 | 23 | root = node(:root, nil, [node(:float, "29382329M")]) 24 | assert parse("29382329M") == root 25 | root = node(:root, nil, [node(:float, "1.33")]) 26 | assert parse("1.33") == root 27 | 28 | root = node(:root, nil, 29 | [node(:symbol, "nilo"), 30 | node(:symbol, "truthy"), 31 | node(:symbol, "falsey")]) 32 | assert parse("nilo truthy falsey") == root 33 | 34 | root = node(:root, nil, 35 | [node(:keyword, "nilo"), 36 | node(:string, "truthy"), 37 | node(:keyword, "falsey")]) 38 | assert parse(":nilo \"truthy\" :falsey") == root 39 | 40 | root = node(:root, nil, 41 | [node(:character, "h"), 42 | node(:character, "i")]) 43 | assert parse("\\h\\i") == root 44 | 45 | root = node(:root, nil, 46 | [node(:string, "Thaïlande")]) 47 | assert parse("\"Thaïlande\"") == root 48 | 49 | assert_raise Ex.UnfinishedTokenError, fn -> 50 | parse(":") 51 | end 52 | 53 | assert_raise Ex.UnfinishedTokenError, fn -> 54 | parse(": :a") 55 | end 56 | end 57 | 58 | test "Map" do 59 | root = node(:root, nil, [node(:map, nil)]) 60 | assert parse("{}") == root 61 | 62 | root = node(:root, nil, 63 | [node(:map, nil, 64 | [node(:keyword, "name"), 65 | node(:string, "John")])]) 66 | assert parse("{:name \"John\"}") == root 67 | 68 | root = node(:root, nil, 69 | [node(:map, nil, 70 | [node(:keyword, "name"), 71 | node(:string, "John"), 72 | node(:keyword, "age"), 73 | node(:integer, "120")])]) 74 | assert parse("{:name \"John\", :age 120}") == root 75 | 76 | assert_raise Ex.OddExpressionCountError, fn -> 77 | parse("{nil true false}") 78 | end 79 | 80 | assert_raise Ex.UnbalancedDelimiterError, fn -> 81 | parse("{nil true ") 82 | end 83 | end 84 | 85 | test "Vector" do 86 | root = node(:root, nil, [node(:vector, nil)]) 87 | assert parse("[]") == root 88 | 89 | root = node(:root, nil, 90 | [node(:vector, nil, 91 | [node(:keyword, "name"), 92 | node(:string, "John")])]) 93 | assert parse("[:name, \"John\"]") == root 94 | 95 | root = node(:root, nil, 96 | [node(:vector, nil, 97 | [node(:keyword, "name"), 98 | node(:string, "John"), 99 | node(:integer, "120")])]) 100 | assert parse("[:name, \"John\", 120]") == root 101 | 102 | assert_raise Ex.UnbalancedDelimiterError, fn -> 103 | parse("[nil true false ") 104 | end 105 | end 106 | 107 | test "List" do 108 | root = node(:root, nil, [node(:list, nil)]) 109 | assert parse("()") == root 110 | 111 | root = node(:root, nil, 112 | [node(:list, nil, 113 | [node(:keyword, "name"), 114 | node(:string, "John")])]) 115 | assert parse("(:name, \"John\")") == root 116 | 117 | root = node(:root, nil, 118 | [node(:list, nil, 119 | [node(:keyword, "name"), 120 | node(:string, "John"), 121 | node(:integer, "120")])]) 122 | assert parse("(:name, \"John\", 120)") == root 123 | 124 | assert_raise Ex.UnbalancedDelimiterError, fn -> 125 | parse("(nil true false ") 126 | end 127 | end 128 | 129 | test "Set" do 130 | root = node(:root, nil, [node(:set, nil)]) 131 | assert parse("#\{}") == root 132 | 133 | root = node(:root, nil, 134 | [node(:set, nil, 135 | [node(:keyword, "name"), 136 | node(:string, "John")])]) 137 | assert parse("#\{:name, \"John\"}") == root 138 | 139 | root = node(:root, nil, 140 | [node(:set, nil, 141 | [node(:keyword, "name"), 142 | node(:string, "John"), 143 | node(:integer, "120")])]) 144 | assert parse("#\{:name, \"John\", 120}") == root 145 | 146 | assert_raise Ex.UnbalancedDelimiterError, fn -> 147 | parse("#\{ nil true false ") 148 | end 149 | end 150 | 151 | test "Tag" do 152 | root = node(:root, nil, 153 | [node(:tag, "inst", 154 | [node(:string, "1985-04-12T23:20:50.52Z")])]) 155 | assert parse("#inst \"1985-04-12T23:20:50.52Z\"") == root 156 | 157 | root = node(:root, nil, 158 | [node(:tag, "some/tag", 159 | [node(:map, nil, 160 | [node(:keyword, "a"), 161 | node(:integer, "1")])])]) 162 | assert parse("#some/tag {:a 1}") == root 163 | 164 | assert_raise Ex.IncompleteTagError, fn -> 165 | parse(":some-keyword #a/tag ") 166 | end 167 | end 168 | 169 | test "Discard" do 170 | root = node(:root, nil, 171 | [node(:set, nil, 172 | [node(:keyword, "name")])]) 173 | assert parse("#\{:name, #_ \"John\"}") == root 174 | 175 | root = node(:root, nil, 176 | [node(:set, nil, 177 | [node(:string, "John"), 178 | node(:integer, "120")])]) 179 | assert parse("#\{#_:name, \"John\", 120}") == root 180 | end 181 | 182 | test "Comment" do 183 | root = node(:root, nil, 184 | [node(:set, nil, 185 | [node(:keyword, "name")])]) 186 | assert parse("#\{:name, \n ;; \"John\" \n}") == root 187 | 188 | root = node(:root, nil, 189 | [node(:set, nil, 190 | [node(:string, "John"), 191 | node(:integer, "120")])]) 192 | assert parse("#\{\n;; :name, \n \"John\", 120}") == root 193 | end 194 | 195 | test "Unexpected Token" do 196 | assert_raise Ex.UnexpectedTokenError, fn -> 197 | parse("#\{\n;; :name, \n \"John\", 120} }") 198 | end 199 | end 200 | 201 | defp node(type, value, children \\ []) do 202 | %Parser.Node{type: type, 203 | value: value, 204 | children: children, 205 | location: nil} 206 | end 207 | end 208 | -------------------------------------------------------------------------------- /test/eden/lexer_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Eden.LexerTest do 2 | use ExUnit.Case 3 | import Eden.Lexer 4 | alias Eden.Lexer 5 | alias Eden.Exception, as: Ex 6 | 7 | test "Whitespace" do 8 | assert tokenize(",,, ") == [] 9 | assert tokenize(" \n \t, \r") == [] 10 | 11 | assert_raise Ex.UnexpectedInputError, fn -> 12 | tokenize(" \n \t, \r a| ,,,") 13 | end 14 | end 15 | 16 | test "nil, true, false" do 17 | assert tokenize("nil") == [token(:nil, "nil")] 18 | assert tokenize(" nil ") == [token(:nil, "nil")] 19 | assert tokenize("true") == [token(:true, "true")] 20 | assert tokenize(" true ") == [token(:true, "true")] 21 | assert tokenize("false") == [token(:false, "false")] 22 | assert tokenize(" false ") == [token(:false, "false")] 23 | 24 | assert List.first(tokenize(" nil{ ")) == token(:nil, "nil") 25 | assert List.first(tokenize(" nilo ")) == token(:symbol, "nilo") 26 | 27 | assert List.first(tokenize(" true} ")) == token(:true, "true") 28 | assert List.first(tokenize(" truedetective ")) == token(:symbol, "truedetective") 29 | 30 | assert List.first(tokenize(" false{ ")) == token(:false, "false") 31 | assert List.first(tokenize(" falsette ")) == token(:symbol, "falsette") 32 | end 33 | 34 | test "String" do 35 | assert tokenize(" \"this is a string\" ") == [token(:string, "this is a string")] 36 | assert tokenize(" \"this is a \\\" string\" ") == [token(:string, "this is a \" string")] 37 | assert_raise Ex.UnfinishedTokenError, fn -> 38 | tokenize(" \"this is an unfinished string ") 39 | end 40 | assert_raise Ex.UnfinishedTokenError, fn -> 41 | tokenize(" \"this is an unfinished string\\\"") 42 | end 43 | end 44 | 45 | test "Character" do 46 | assert tokenize(" \\t ") == [token(:character, "t")] 47 | assert tokenize(" \\r,, ") == [token(:character, "r")] 48 | end 49 | 50 | test "Keyword" do 51 | assert tokenize(" :a-keyword ") == [token(:keyword, "a-keyword")] 52 | assert tokenize(":a-keyword") == [token(:keyword, "a-keyword")] 53 | assert tokenize(" :question? ") == [token(:keyword, "question?")] 54 | assert tokenize(":question?{") == [token(:keyword, "question?"), token(:curly_open, "{")] 55 | assert tokenize(":k?+._-!7><$&=*") == [token(:keyword, "k?+._-!7><$&=*")] 56 | 57 | assert_raise Ex.UnexpectedInputError, fn -> 58 | tokenize(" :question?\\") 59 | end 60 | end 61 | 62 | test "Symbol" do 63 | assert tokenize(" a-keyword ") == [token(:symbol, "a-keyword")] 64 | assert tokenize("a-keyword") == [token(:symbol, "a-keyword")] 65 | assert tokenize(" question? ") == [token(:symbol, "question?")] 66 | assert tokenize("question?{") == [token(:symbol, "question?"), token(:curly_open, "{")] 67 | assert tokenize("k?+._-!7><$&=*") == [token(:symbol, "k?+._-!7><$&=*")] 68 | assert tokenize("ns/name") == [token(:symbol, "ns/name")] 69 | 70 | assert_raise Ex.UnexpectedInputError, fn -> 71 | tokenize(" question?\\") 72 | end 73 | assert_raise Ex.UnexpectedInputError, fn -> 74 | tokenize("ns/name/ss") 75 | end 76 | end 77 | 78 | test "Integer" do 79 | assert tokenize("1234") == [token(:integer, "1234")] 80 | assert tokenize("-1234") == [token(:integer, "-1234")] 81 | assert tokenize("+1234") == [token(:integer, "+1234")] 82 | assert tokenize(" 1234 ") == [token(:integer, "1234")] 83 | assert tokenize("1234N") == [token(:integer, "1234N")] 84 | assert tokenize("1234N{") == [token(:integer, "1234N"), token(:curly_open, "{")] 85 | 86 | assert_raise Ex.UnexpectedInputError, fn -> 87 | assert tokenize("1234a") 88 | end 89 | end 90 | 91 | test "Float" do 92 | assert tokenize("1234.12") == [token(:float, "1234.12")] 93 | assert tokenize(" 1234.12 ") == [token(:float, "1234.12")] 94 | assert tokenize("1234M") == [token(:float, "1234M")] 95 | assert tokenize("1234M{") == [token(:float, "1234M"), token(:curly_open, "{")] 96 | 97 | assert tokenize("1234E12") == [token(:float, "1234E12")] 98 | assert tokenize("1234E-12") == [token(:float, "1234E-12")] 99 | assert tokenize("1234E+12") == [token(:float, "1234E+12")] 100 | 101 | assert tokenize("1234e12") == [token(:float, "1234e12")] 102 | assert tokenize("1234e-12") == [token(:float, "1234e-12")] 103 | assert tokenize("1234e+12") == [token(:float, "1234e+12")] 104 | 105 | assert_raise Ex.UnexpectedInputError, fn -> 106 | assert tokenize("1234.a") 107 | end 108 | assert_raise Ex.UnexpectedInputError, fn -> 109 | assert tokenize("1234.121a ") 110 | end 111 | assert_raise Ex.UnexpectedInputError, fn -> 112 | assert tokenize("1234E0a1") 113 | end 114 | assert_raise Ex.UnfinishedTokenError, fn -> 115 | tokenize("1234E") 116 | end 117 | assert_raise Ex.UnfinishedTokenError, fn -> 118 | tokenize("1234.") 119 | end 120 | assert_raise Ex.UnfinishedTokenError, fn -> 121 | tokenize("1234. :kw") 122 | end 123 | end 124 | 125 | test "Delimiters" do 126 | assert tokenize("{[#\{}]} )()") == [token(:curly_open, "{"), 127 | token(:bracket_open, "["), 128 | token(:set_open, "#\{"), 129 | token(:curly_close, "}"), 130 | token(:bracket_close, "]"), 131 | token(:curly_close, "}"), 132 | token(:paren_close, ")"), 133 | token(:paren_open, "("), 134 | token(:paren_close, ")")] 135 | end 136 | 137 | test "Discard" do 138 | assert tokenize("#_ ") == [token(:discard, "#_")] 139 | assert tokenize("1 #_ :kw") == [token(:integer, "1"), 140 | token(:discard, "#_"), 141 | token(:keyword, "kw")] 142 | end 143 | 144 | test "Tag" do 145 | assert tokenize("#ns/name") == [token(:tag, "ns/name")] 146 | assert tokenize("#whatever") == [token(:tag, "whatever")] 147 | assert tokenize(" #whatever :kw") == [token(:tag, "whatever"), 148 | token(:keyword, "kw")] 149 | end 150 | 151 | test "Comment" do 152 | assert tokenize("1 ;; hello") == [token(:integer, "1"), 153 | token(:comment, " hello")] 154 | assert tokenize("1 ;; hello\n\r") == [token(:integer, "1"), 155 | token(:comment, " hello")] 156 | assert tokenize("1 ;; hello\n\r bla") == [token(:integer, "1"), 157 | token(:comment, " hello"), 158 | token(:symbol, "bla")] 159 | end 160 | 161 | test "Line and Column Information" do 162 | tokens = [token(:integer, "1", %{line: 1, col: 0}), 163 | token(:comment, " hello", %{line: 1, col: 2})] 164 | assert tokenize("1 ;; hello", location: true) == tokens 165 | assert tokenize("1 ;; hello\r\n", location: true) == tokens 166 | 167 | tokens = [token(:integer, "1", %{line: 1, col: 0}), 168 | token(:comment, " hello", %{line: 1, col: 2}), 169 | token(:symbol, "bla", %{line: 2, col: 1})] 170 | assert tokenize("1 ;; hello\r\n bla", location: true) == tokens 171 | assert tokenize("1 ;; hello\n\r bla", location: true) == tokens 172 | 173 | tokens = [token(:integer, "1", %{line: 1, col: 0}), 174 | token(:string, "hello \n world", %{line: 2, col: 0}), 175 | token(:keyword, "kw", %{line: 3, col: 8})] 176 | assert tokenize("1 \n\"hello \n world\" :kw ", location: true) == tokens 177 | 178 | tokens = [token(:integer, "1", %{line: 1, col: 0}), 179 | token(:string, "hello \n \" world", %{line: 2, col: 0}), 180 | token(:keyword, "kw", %{line: 3, col: 1})] 181 | assert tokenize("1 \n\"hello \\n \\\" world\"\n :kw ", location: true) == tokens 182 | end 183 | 184 | defp token(type, value, location \\ nil) do 185 | token = %Lexer.Token{type: type, value: value} 186 | if location, 187 | do: Map.put(token, :location, location), 188 | else: token 189 | end 190 | end 191 | -------------------------------------------------------------------------------- /lib/eden/parser.ex: -------------------------------------------------------------------------------- 1 | defmodule Eden.Parser do 2 | alias Eden.Lexer 3 | alias Eden.Parser.Node 4 | alias Eden.Exception, as: Ex 5 | 6 | @moduledoc """ 7 | Provides a single function that returns an `Eden.Parser.Node` 8 | struct, which is the `:root` node of the parse tree. 9 | """ 10 | 11 | require Logger 12 | 13 | @doc """ 14 | Takes a string and returns the root node of the parse tree. 15 | 16 | All nodes are an `Eden.Parser.Node` struct, each of which has a `:type`, 17 | a `:value` and an optional `:location` property. 18 | 19 | The returned node is always of type `:root`, whose children are all the 20 | expressions found in the string provided. 21 | 22 | Options: 23 | 24 | - `:location` - a `boolean` that determines if nodes should include row and column information. 25 | 26 | ## Examples 27 | 28 | iex> Eden.Parser.parse("nil") 29 | :root 30 | :nil "nil" 31 | 32 | iex> Eden.Parse.parse("nil", location: true) 33 | :root 34 | :nil "nil" (1,0) 35 | 36 | iex> Eden.Parse.parse("[1 2 3]") 37 | :root 38 | :vector 39 | :integer "1" 40 | :integer "2" 41 | :integer "3" 42 | 43 | iex> Eden.Parse.parse("[1 2 3]", location: true) 44 | :root 45 | :vector (1,0) 46 | :integer "1" (1,1) 47 | :integer "2" (1,3) 48 | :integer "3" (1,5) 49 | """ 50 | def parse(input, opts \\ [location: false]) when is_binary(input) do 51 | tokens = Lexer.tokenize(input, opts) 52 | state = %{tokens: tokens, node: new_node(:root)} 53 | state = exprs(state) 54 | if not Enum.empty?(state.tokens) do 55 | raise Ex.UnexpectedTokenError, List.first(state.tokens) 56 | end 57 | Node.reverse_children(state.node) 58 | end 59 | 60 | ############################################################################## 61 | ## Rules and Productions 62 | ############################################################################## 63 | 64 | defp exprs(state) do 65 | state 66 | |> expr 67 | |> skip_when(&exprs/1, &is_nil/1) 68 | |> return_when(state, &is_nil/1) 69 | end 70 | 71 | defp expr(%{tokens: []}) do 72 | nil 73 | end 74 | defp expr(state) do 75 | terminal(state, :nil) 76 | || terminal(state, :false) 77 | || terminal(state, :true) 78 | || terminal(state, :symbol) 79 | || terminal(state, :keyword) 80 | || terminal(state, :integer) 81 | || terminal(state, :float) 82 | || terminal(state, :string) 83 | || terminal(state, :character) 84 | || map_begin(state) 85 | || vector_begin(state) 86 | || list_begin(state) 87 | || set_begin(state) 88 | || tag(state) 89 | || discard(state) 90 | || comment(state) 91 | end 92 | 93 | defp terminal(state, type) do 94 | {state, token} = pop_token(state) 95 | if token?(token, type) do 96 | node = new_node(type, token, true) 97 | add_node(state, node) 98 | end 99 | end 100 | 101 | ## Map 102 | 103 | defp map_begin(state) do 104 | {state, token} = pop_token(state) 105 | if token?(token, :curly_open) do 106 | state 107 | |> set_node(new_node(:map, token)) 108 | |> pairs 109 | |> map_end 110 | |> restore_node(state) 111 | end 112 | end 113 | 114 | defp pairs(state) do 115 | state 116 | |> pair 117 | |> skip_when(&pairs/1, &is_nil/1) 118 | |> return_when(state, &is_nil/1) 119 | end 120 | 121 | defp pair(state) do 122 | state 123 | |> expr 124 | |> skip_when(&pair2/1, &is_nil/1) 125 | end 126 | 127 | defp pair2(state) do 128 | state 129 | |> expr 130 | |> raise_when(Ex.OddExpressionCountError, state, &is_nil/1) 131 | end 132 | 133 | defp map_end(state) do 134 | {state, token} = pop_token(state) 135 | if not token?(token, :curly_close) do 136 | raise Ex.UnbalancedDelimiterError, state.node 137 | end 138 | state 139 | end 140 | 141 | ## Vector 142 | 143 | defp vector_begin(state) do 144 | {state, token} = pop_token(state) 145 | if token?(token, :bracket_open) do 146 | state 147 | |> set_node(new_node(:vector, token)) 148 | |> exprs 149 | |> vector_end 150 | |> restore_node(state) 151 | end 152 | end 153 | 154 | defp vector_end(state) do 155 | {state, token} = pop_token(state) 156 | if not token?(token, :bracket_close) do 157 | raise Ex.UnbalancedDelimiterError, state.node 158 | end 159 | state 160 | end 161 | 162 | ## List 163 | 164 | defp list_begin(state) do 165 | {state, token} = pop_token(state) 166 | if token?(token, :paren_open) do 167 | state 168 | |> set_node(new_node(:list, token)) 169 | |> exprs 170 | |> list_end 171 | |> restore_node(state) 172 | end 173 | end 174 | 175 | defp list_end(state) do 176 | {state, token} = pop_token(state) 177 | if not token?(token, :paren_close) do 178 | raise Ex.UnbalancedDelimiterError, state.node 179 | end 180 | state 181 | end 182 | 183 | ## Set 184 | 185 | defp set_begin(state) do 186 | {state, token} = pop_token(state) 187 | if token?(token, :set_open) do 188 | state 189 | |> set_node(new_node(:set, token)) 190 | |> exprs 191 | |> set_end 192 | |> restore_node(state) 193 | end 194 | end 195 | 196 | defp set_end(state) do 197 | {state, token} = pop_token(state) 198 | if not token?(token, :curly_close) do 199 | raise Ex.UnbalancedDelimiterError, state.node 200 | end 201 | state 202 | end 203 | 204 | ## Tag 205 | 206 | defp tag(state) do 207 | {state, token} = pop_token(state) 208 | if token?(token, :tag) do 209 | node = new_node(:tag, token, true) 210 | state 211 | |> set_node(node) 212 | |> expr 213 | |> raise_when(Ex.IncompleteTagError, node, &is_nil/1) 214 | |> restore_node(state) 215 | end 216 | end 217 | 218 | ## Discard 219 | 220 | defp discard(state) do 221 | {state, token} = pop_token(state) 222 | if token?(token, :discard) do 223 | state 224 | |> set_node(new_node(:discard, token)) 225 | |> expr 226 | |> raise_when(Ex.MissingDiscardExpressionError, state, &is_nil/1) 227 | |> restore_node(state, false) 228 | end 229 | end 230 | 231 | ## Comment 232 | 233 | defp comment(state) do 234 | {state, token} = pop_token(state) 235 | if token?(token, :comment) do 236 | state 237 | end 238 | end 239 | 240 | ############################################################################## 241 | ## Helper functions 242 | ############################################################################## 243 | 244 | ## Node 245 | 246 | defp new_node(type, token \\ nil, use_value? \\ false) do 247 | location = if token && Map.has_key?(token, :location) do 248 | token.location 249 | end 250 | value = if token && use_value? do 251 | token.value 252 | end 253 | %Node{type: type, 254 | location: location, 255 | value: value, 256 | children: []} 257 | end 258 | 259 | defp add_node(state, node) do 260 | update_in(state, [:node, :children], fn children -> 261 | [node | children] 262 | end) 263 | end 264 | 265 | defp set_node(state, node) do 266 | Map.put(state, :node, node) 267 | end 268 | 269 | defp restore_node(new_state, old_state, add_child? \\ true) do 270 | child_node = Node.reverse_children(new_state.node) 271 | old_state = Map.put(new_state, :node, old_state.node) 272 | if add_child? do 273 | add_node(old_state, child_node) 274 | else 275 | old_state 276 | end 277 | end 278 | 279 | ## Token 280 | 281 | defp token?(nil, _), do: false 282 | defp token?(token, type), do: (token.type == type) 283 | 284 | defp pop_token(state) do 285 | {update_in(state, [:tokens], &tail/1), 286 | List.first(state.tokens)} 287 | end 288 | 289 | ## Utils 290 | 291 | defp return_when(x, y, pred?) do 292 | if(pred?.(x), do: y, else: x) 293 | end 294 | 295 | defp skip_when(x, fun, pred?) do 296 | if(pred?.(x), do: x, else: fun.(x)) 297 | end 298 | 299 | defp raise_when(x, ex, msg, pred?) do 300 | if(pred?.(x), do: (raise ex, msg), else: x) 301 | end 302 | 303 | defp tail([]), do: [] 304 | defp tail(list), do: tl(list) 305 | end 306 | -------------------------------------------------------------------------------- /lib/eden/lexer.ex: -------------------------------------------------------------------------------- 1 | defmodule Eden.Lexer do 2 | alias Eden.Exception, as: Ex 3 | @moduledoc """ 4 | A module that implements a lexer for the edn format through its 5 | only function `tokenize/1`. 6 | """ 7 | 8 | defmodule Token do 9 | defstruct type: nil, value: nil 10 | end 11 | 12 | ############################################################################## 13 | ## API 14 | ############################################################################## 15 | 16 | @doc """ 17 | Takes a string and returns a list of tokens. 18 | 19 | Options: 20 | 21 | - `:location` - a `boolean` that determines wether the location information 22 | should be included with each token. Lines are one-based and columns are 23 | zero-based. The default value for `:location` is `false`. 24 | 25 | ## Examples 26 | 27 | iex> Eden.Lexer.tokenize("nil") 28 | [%Eden.Lexer.Token{type: nil, value: "nil"}] 29 | 30 | iex> Eden.Lexer.tokenize("nil", location: true) 31 | [%Eden.Lexer.Token{location: %{col: 0, line: 1}, type: nil, value: "nil"}] 32 | """ 33 | def tokenize(input, opts \\ [location: false]) do 34 | initial_state = %{state: :new, 35 | tokens: [], 36 | current: nil, 37 | opts: opts, 38 | location: %{line: 1, col: 0}} 39 | _tokenize(initial_state, input) 40 | end 41 | 42 | ############################################################################## 43 | ## Private functions 44 | ############################################################################## 45 | 46 | # End of input 47 | defp _tokenize(state, <<>>) do 48 | state 49 | |> valid? 50 | |> add_token(state.current) 51 | |> Map.get(:tokens) 52 | |> Enum.reverse 53 | end 54 | 55 | # Comment 56 | defp _tokenize(state = %{state: :new}, <<";" :: utf8, rest :: binary>>) do 57 | token = token(:comment, "") 58 | start_token(state, :comment, token, ";", rest) 59 | end 60 | defp _tokenize(state = %{state: :comment}, <>) 61 | when <> in ["\n", "\r"] do 62 | end_token(state, <>, rest) 63 | end 64 | defp _tokenize(state = %{state: :comment}, <<";" :: utf8, rest :: binary>>) do 65 | skip_char(state, ";", rest) 66 | end 67 | defp _tokenize(state = %{state: :comment}, <>) do 68 | consume_char(state, <>, rest) 69 | end 70 | 71 | # Literals 72 | defp _tokenize(state = %{state: :new}, <<"nil" :: utf8, rest :: binary>>) do 73 | token = token(:nil, "nil") 74 | start_token(state, :check_literal, token, "nil", rest) 75 | end 76 | defp _tokenize(state = %{state: :new}, <<"true" :: utf8, rest :: binary>>) do 77 | token = token(:true, "true") 78 | start_token(state, :check_literal, token, "true", rest) 79 | end 80 | defp _tokenize(state = %{state: :new}, <<"false" :: utf8, rest :: binary>>) do 81 | token = token(:false, "false") 82 | start_token(state, :check_literal, token, "false", rest) 83 | end 84 | defp _tokenize(state = %{state: :check_literal}, <> = input) do 85 | if separator?(<>) do 86 | end_token(state, "", input) 87 | else 88 | token = token(:symbol, state.current.value) 89 | start_token(state, :symbol, token, "", input) 90 | end 91 | end 92 | 93 | # String 94 | defp _tokenize(state = %{state: :new}, <<"\"" :: utf8, rest :: binary>>) do 95 | token = token(:string, "") 96 | start_token(state, :string, token, "\"", rest) 97 | end 98 | defp _tokenize(state = %{state: :string}, <<"\\" :: utf8, char :: utf8, rest :: binary>>) do 99 | # TODO: this will cause the line count to get corrupted, 100 | # either use the original or send the real content as 101 | # an optional argument. 102 | consume_char(state, escaped_char(<>), rest, <<"\\" :: utf8, char :: utf8>>) 103 | end 104 | defp _tokenize(state = %{state: :string}, <<"\"" :: utf8, rest :: binary>>) do 105 | end_token(state, "\"", rest) 106 | end 107 | defp _tokenize(state = %{state: :string}, <>) do 108 | consume_char(state, <>, rest) 109 | end 110 | 111 | # Character 112 | defp _tokenize(state = %{state: :new}, <<"\\" :: utf8, char :: utf8, rest :: binary>>) do 113 | token = token(:character, <>) 114 | end_token(state, token, "\\" <> <>, rest) 115 | end 116 | 117 | # Keyword and Symbol 118 | defp _tokenize(state = %{state: :new}, <<":" :: utf8, rest :: binary>>) do 119 | token = token(:keyword, "") 120 | start_token(state, :symbol, token, ":", rest) 121 | end 122 | defp _tokenize(state = %{state: :symbol}, <<"/" :: utf8, rest :: binary>>) do 123 | if not String.contains?(state.current.value, "/") do 124 | consume_char(state, <<"/" :: utf8>>, rest) 125 | else 126 | raise Ex.UnexpectedInputError, "/" 127 | end 128 | end 129 | defp _tokenize(state = %{state: :symbol}, <> = input) do 130 | if symbol_char?(<>) do 131 | consume_char(state, <>, rest) 132 | else 133 | end_token(state, "", input) 134 | end 135 | end 136 | 137 | # Integers & Float 138 | defp _tokenize(state = %{state: :new}, <>) 139 | when <> in ["-", "+"] do 140 | token = token(:integer, <>) 141 | start_token(state, :number, token, <>, rest) 142 | end 143 | defp _tokenize(state = %{state: :exponent}, <>) 144 | when <> in ["-", "+"] do 145 | consume_char(state, <>, rest) 146 | end 147 | defp _tokenize(state = %{state: :number}, <<"N" :: utf8, rest :: binary>>) do 148 | state = append_to_current(state, "N") 149 | end_token(state, "N", rest) 150 | end 151 | defp _tokenize(state = %{state: :number}, <<"M" :: utf8, rest :: binary>>) do 152 | state = append_to_current(state, "M") 153 | token = token(:float, state.current.value) 154 | end_token(state, token, "M", rest) 155 | end 156 | defp _tokenize(state = %{state: :number}, <<"." :: utf8, rest :: binary>>) do 157 | state = append_to_current(state, ".") 158 | token = token(:float, state.current.value) 159 | start_token(state, :fraction, token, ".", rest) 160 | end 161 | defp _tokenize(state = %{state: :number}, <>) 162 | when <> in ["e", "E"] do 163 | state = append_to_current(state, <>) 164 | token = token(:float, state.current.value) 165 | start_token(state, :exponent, token, <>, rest) 166 | end 167 | defp _tokenize(state = %{state: s}, <> = input) 168 | when s in [:number, :exponent, :fraction] do 169 | cond do 170 | digit?(<>) -> 171 | state 172 | |> set_state(:number) 173 | |> consume_char(<>, rest) 174 | s in [:exponent, :fraction] and separator?(<>) -> 175 | raise Ex.UnfinishedTokenError, state.current 176 | separator?(<>) -> 177 | end_token(state, "", input) 178 | true -> 179 | raise Ex.UnexpectedInputError, <> 180 | end 181 | end 182 | 183 | # Delimiters 184 | defp _tokenize(state = %{state: :new}, <>) 185 | when <> in ["{", "}", "[", "]", "(", ")"] do 186 | delim = <> 187 | token = token(delim_type(delim), delim) 188 | end_token(state, token, delim, rest) 189 | end 190 | defp _tokenize(state = %{state: :new}, <<"#\{" :: utf8, rest :: binary>>) do 191 | token = token(:set_open, "#\{") 192 | end_token(state, token, "#\{", rest) 193 | end 194 | 195 | # Whitespace 196 | defp _tokenize(state = %{state: :new}, <>) 197 | when <> in [" ", "\t", "\r", "\n", ","] do 198 | skip_char(state, <>, rest) 199 | end 200 | 201 | # Discard 202 | defp _tokenize(state = %{state: :new}, <<"#_" :: utf8, rest :: binary>>) do 203 | token = token(:discard, "#_") 204 | end_token(state, token, "#_", rest) 205 | end 206 | 207 | # Tags 208 | defp _tokenize(state = %{state: :new}, <<"#" :: utf8, rest :: binary>>) do 209 | token = token(:tag, "") 210 | start_token(state, :symbol, token, "#", rest) 211 | end 212 | 213 | # Symbol, Integer or Invalid input 214 | defp _tokenize(state = %{state: :new}, <>) do 215 | cond do 216 | alpha?(<>) -> 217 | token = token(:symbol, <>) 218 | start_token(state, :symbol, token, <>, rest) 219 | digit?(<>) -> 220 | token = token(:integer, <>) 221 | start_token(state, :number, token, <>, rest) 222 | true -> 223 | raise Ex.UnexpectedInputError, <> 224 | end 225 | end 226 | 227 | # Unexpected Input 228 | defp _tokenize(_, <>) do 229 | raise Ex.UnexpectedInputError, <> 230 | end 231 | 232 | ############################################################################## 233 | ## Helper functions 234 | ############################################################################## 235 | 236 | defp start_token(state, name, token, char, rest) do 237 | state 238 | |> set_state(name) 239 | |> set_token(token) 240 | |> update_location(char) 241 | |> _tokenize(rest) 242 | end 243 | 244 | defp consume_char(state, char, rest, real_char \\ nil) when is_binary(char) do 245 | state 246 | |> update_location(real_char || char) 247 | |> append_to_current(char) 248 | |> _tokenize(rest) 249 | end 250 | 251 | defp skip_char(state, char, rest) when is_binary(char) do 252 | state 253 | |> update_location(char) 254 | |> _tokenize(rest) 255 | end 256 | 257 | defp end_token(state, char, rest) do 258 | state 259 | |> update_location(char) 260 | |> add_token(state.current) 261 | |> reset 262 | |> _tokenize(rest) 263 | end 264 | 265 | defp end_token(state, token, char, rest) do 266 | state 267 | |> set_token(token) 268 | |> end_token(char, rest) 269 | end 270 | 271 | defp update_location(state, "") do 272 | state 273 | end 274 | defp update_location(state, <<"\n" :: utf8, rest :: binary>>) do 275 | state 276 | |> put_in([:location, :line], state.location.line + 1) 277 | |> put_in([:location, :col], 0) 278 | |> update_location(rest) 279 | end 280 | defp update_location(state, <<"\r" :: utf8, rest :: binary>>) do 281 | update_location(state, rest) 282 | end 283 | defp update_location(state, <<_ :: utf8, rest :: binary>>) do 284 | state 285 | |> put_in([:location, :col], state.location.col + 1) 286 | |> update_location(rest) 287 | end 288 | 289 | defp token(type, value) do 290 | %Token{type: type, value: value} 291 | end 292 | 293 | defp set_token(state, token) do 294 | token = if state.opts[:location] do 295 | Map.put(token, :location, state.location) 296 | else 297 | token 298 | end 299 | Map.put(state, :current, token) 300 | end 301 | 302 | defp set_state(state, name) do 303 | Map.put(state, :state, name) 304 | end 305 | 306 | defp append_to_current(%{current: current} = state, c) do 307 | current = %{current | value: current.value <> c} 308 | %{state | current: current} 309 | end 310 | 311 | defp reset(state) do 312 | %{state | 313 | state: :new, 314 | current: nil} 315 | end 316 | 317 | defp valid?(%{state: state, current: current}) 318 | when state in [:string, :exponent, :character, :fraction] do 319 | raise Ex.UnfinishedTokenError, current 320 | end 321 | defp valid?(state) do 322 | state 323 | end 324 | 325 | defp add_token(state, nil) do 326 | state 327 | end 328 | defp add_token(state, token) do 329 | if token.type == :keyword and token.value == "" do 330 | raise Ex.UnfinishedTokenError, token 331 | end 332 | %{state | tokens: [token | state.tokens]} 333 | end 334 | 335 | defp delim_type("{"), do: :curly_open 336 | defp delim_type("}"), do: :curly_close 337 | defp delim_type("["), do: :bracket_open 338 | defp delim_type("]"), do: :bracket_close 339 | defp delim_type("("), do: :paren_open 340 | defp delim_type(")"), do: :paren_close 341 | 342 | defp escaped_char("\""), do: "\"" 343 | defp escaped_char("t"), do: "\t" 344 | defp escaped_char("r"), do: "\r" 345 | defp escaped_char("n"), do: "\n" 346 | defp escaped_char("\\"), do: "\\" 347 | 348 | defp alpha?(char), do: String.match?(char, ~r/[a-zA-Z]/) 349 | 350 | defp digit?(char), do: String.match?(char, ~r/[0-9]/) 351 | 352 | defp symbol_char?(char), do: String.match?(char, ~r/[_?a-zA-Z0-9.*+!\-$%&=<>\#:]/) 353 | 354 | defp whitespace?(char), do: String.match?(char, ~r/[\s,]/) 355 | 356 | defp delim?(char), do: String.match?(char, ~r/[\{\}\[\]\(\)]/) 357 | 358 | defp separator?(char), do: whitespace?(char) or delim?(char) 359 | end 360 | --------------------------------------------------------------------------------