├── .buildkite └── pipelines │ ├── common.sh │ ├── dinerl-merge-builder │ └── start.sh │ └── dinerl-pr-builder │ └── start.yml ├── .gitignore ├── CHANGELOG.md ├── LICENSE ├── Makefile ├── README.md ├── elvis.config ├── rebar.config ├── rebar.lock └── src ├── dinerl.app.src ├── dinerl.erl ├── dinerl_client.erl ├── dinerl_util.erl ├── dmochijson2.erl ├── dmochinum.erl └── dynamodb.erl /.buildkite/pipelines/common.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | function die() { 5 | echo ${1} 6 | exit 254 7 | } 8 | 9 | function call_bk() { 10 | method=${1} 11 | path=${2} 12 | qs="" 13 | if [ ! -z ${3+x} ]; then 14 | qs=${3} 15 | fi 16 | url=$(bk_url ${path} ${qs}) 17 | curl -sX ${method} "${url}" 18 | } 19 | 20 | function bk_url() { 21 | path=${1} 22 | qs="" 23 | if [ ! -z ${2+x} ]; then 24 | qs=${2} 25 | fi 26 | echo "https://api.buildkite.com/v2/organizations/adroll-group/${path}?access_token=${BUILDKITE_TOKEN}&${qs}" 27 | } 28 | -------------------------------------------------------------------------------- /.buildkite/pipelines/dinerl-merge-builder/start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -ue 3 | 4 | root=$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )/../.. 5 | . ${root}/pipelines/common.sh 6 | 7 | if [ "${BUILDKITE_BRANCH}" = "main" ]; then 8 | cat <>, [{<<"HashKeyElement">>, [{<<"AttributeName">>, <<"Key">>}, {<<"AttributeType">>, <<"S">>}]}], 50, 50). 19 | dinerl:list_tables(). 20 | dinerl:put_item(<<"TestTable">>, [{<<"Key">>, [{<<"S">>, <<"jello">>}]}], []). 21 | dinerl:get_item(<<"TestTable">>, [{<<"HashKeyElement">>, [{<<"S">>, <<"jello">>}]}], []). 22 | 23 | put(Key, Value, TTL, Now) -> 24 | dinerl:put_item(<<"Attributions">>, [{<<"UserKey">>, [{<<"S">>, Key}]}, 25 | {<<"Updated">>, [{<<"N">>, list_to_binary(integer_to_list(Now))}]}, 26 | {<<"TTL">>, [{<<"N">>, list_to_binary(integer_to_list(TTL))}]}, 27 | {<<"Value">>, [{<<"S">>, Value}]}], []). 28 | 29 | get(Key, Now, Default) -> 30 | case dinerl:get_item(<<"Attributions">>, [{<<"HashKeyElement">>, [{<<"S">>, Key}]}], [{attrs, [<<"TTL">>, <<"Updated">>, <<"Value">>, <<"Visited">>]}]) of 31 | {ok, Element} -> 32 | ParsedResult = parsejson(Element), 33 | return_if_not_expired(Key, ParsedResult, Now, Default); 34 | 35 | {error, Short, Long} -> 36 | io:format("~p", [Long]), 37 | Default 38 | end. 39 | 40 | add(Key, Value, TTL, Now) -> 41 | dinerl:update_item(<<"Attributions">>, 42 | [{<<"HashKeyElement">>, [{<<"S">>, Key}]}], 43 | [{update, [{<<"Visited">>, [{value, [{<<"SS">>, [Value]}]}, 44 | {action, add}]}, 45 | {<<"Updated">>, [{value, [{<<"N">>, list_to_binary(integer_to_list(Now))}]}, 46 | {action, put}]}, 47 | {<<"TTL">>, [{value, [{<<"N">>, list_to_binary(integer_to_list(TTL))}]}, 48 | {action, put}]}]}]). 49 | 50 | parsejson([]) -> 51 | []; 52 | parsejson({struct, L}) -> 53 | parsejson(L); 54 | parsejson([{<<"Item">>, {struct, Fields}}|_Rest]) -> 55 | parsejsonfields(Fields, []); 56 | parsejson([_H|T]) -> 57 | parsejson(T). 58 | 59 | parsejsonfields([], Acc) -> 60 | Acc; 61 | parsejsonfields([{Name, {struct, [{<<"N">>, Value}]}}|Rest], Acc) -> 62 | parsejsonfields(Rest, [{Name, list_to_integer(binary_to_list(Value))}|Acc]); 63 | parsejsonfields([{Name, {struct, [{<<"S">>, Value}]}}|Rest], Acc) -> 64 | parsejsonfields(Rest, [{Name, Value}|Acc]); 65 | parsejsonfields([{Name, {struct, [{<<"NS">>, Value}]}}|Rest], Acc) -> 66 | parsejsonfields(Rest, [{Name, all_to_int(Value)}|Acc]); 67 | parsejsonfields([{Name, {struct, [{<<"SS">>, Value}]}}|Rest], Acc) -> 68 | parsejsonfields(Rest, [{Name, Value}|Acc]). 69 | 70 | return_if_not_expired(_, [], _, Default) -> 71 | Default; 72 | return_if_not_expired(Key, ParsedResult, Now, Default) -> 73 | Updated = proplists:get_value(<<"Updated">>, ParsedResult), 74 | TTL = proplists:get_value(<<"TTL">>, ParsedResult), 75 | case (Updated+TTL) > Now of 76 | true -> 77 | ParsedResult; 78 | false -> 79 | dinerl:delete_item(<<"Attributions">>, [{<<"HashKeyElement">>, [{<<"S">>, Key}]}], []), 80 | Default 81 | end. 82 | 83 | all_to_int(L) -> 84 | all_to_int(L, []). 85 | all_to_int([], Acc) -> 86 | lists:reverse(Acc); 87 | all_to_int([H|T], Acc) -> 88 | all_to_int(T, [list_to_integer(binary_to_list(H))|Acc]). 89 | 90 | pytime() -> 91 | pytime(erlang:now()). 92 | pytime({MegaSecs, Secs, MicroSecs}) -> 93 | erlang:trunc((1.0e+6 * MegaSecs) + Secs + (1.0e-6 * MicroSecs)). 94 | ``` 95 | -------------------------------------------------------------------------------- /elvis.config: -------------------------------------------------------------------------------- 1 | [{elvis, 2 | [{config, 3 | [#{dirs => ["src"], 4 | filter => "*.erl", 5 | ruleset => erl_files, 6 | rules => 7 | [{elvis_text_style, line_length, #{limit => 120}}, 8 | {elvis_style, god_modules, #{limit => 50}}, 9 | {elvis_style, dont_repeat_yourself, #{min_complexity => 20}}, 10 | {elvis_style, nesting_level, #{level => 4}}, 11 | {elvis_style, atom_naming_convention, disable}, 12 | {elvis_style, macro_names, disable}]}]}]}]. 13 | -------------------------------------------------------------------------------- /rebar.config: -------------------------------------------------------------------------------- 1 | {erl_opts, 2 | [warn_unused_import, warn_export_vars, warnings_as_errors, verbose, report, debug_info]}. 3 | 4 | {deps, [{lhttpc, "1.4.0", {pkg, nextroll_lhttpc}}, {erliam, "1.0.1"}]}. 5 | 6 | {cover_enabled, true}. 7 | 8 | {cover_opts, [verbose]}. 9 | 10 | {project_plugins, 11 | [{rebar3_hex, "~> 7.0.7"}, 12 | {rebar3_format, "~> 1.3.0"}, 13 | {rebar3_lint, "~> 3.2.3"}, 14 | {rebar3_hank, "~> 1.4.0"}]}. 15 | 16 | {dialyzer, 17 | [{warnings, [unknown, no_return, error_handling]}, 18 | {get_warnings, true}, 19 | {plt_apps, top_level_deps}, 20 | {plt_extra_apps, []}, 21 | {plt_location, local}, 22 | {base_plt_apps, [erts, stdlib, kernel]}, 23 | {base_plt_location, global}]}. 24 | 25 | {xref_checks, [undefined_function_calls, locals_not_used, deprecated_function_calls]}. 26 | 27 | {eunit_opts, [verbose, {report, {eunit_surefire, [{dir, "."}]}}]}. 28 | 29 | {alias, [{test, [format, lint, hank, xref, dialyzer, eunit, cover]}]}. 30 | -------------------------------------------------------------------------------- /rebar.lock: -------------------------------------------------------------------------------- 1 | {"1.2.0", 2 | [{<<"erliam">>,{pkg,<<"erliam">>,<<"1.0.1">>},0}, 3 | {<<"jiffy">>,{pkg,<<"jiffy">>,<<"1.1.1">>},1}, 4 | {<<"lhttpc">>,{pkg,<<"nextroll_lhttpc">>,<<"1.4.0">>},0}]}. 5 | [ 6 | {pkg_hash,[ 7 | {<<"erliam">>, <<"20E1ECB876AFDEEC2DE07483E2D174B1E3DB38848ED981145DAB9A889E7B55F9">>}, 8 | {<<"jiffy">>, <<"ACA10F47AA91697BF24AB9582C74E00E8E95474C7EF9F76D4F1A338D0F5DE21B">>}, 9 | {<<"lhttpc">>, <<"45282FF22BC55E6AE751CF87AC42C261DC4FAAFADD9C034E127ECED74E672FAB">>}]}, 10 | {pkg_hash_ext,[ 11 | {<<"erliam">>, <<"2EE375544AC36711BEEB5EC56DB060488447CECC308763BC8B4A4FEE894AAF76">>}, 12 | {<<"jiffy">>, <<"62E1F0581C3C19C33A725C781DFA88410D8BFF1BBAFC3885A2552286B4785C4C">>}, 13 | {<<"lhttpc">>, <<"57BA3D5720FBD17C75D8563169394B5F6CD160161D64A8A9F96F7E829221C648">>}]} 14 | ]. 15 | -------------------------------------------------------------------------------- /src/dinerl.app.src: -------------------------------------------------------------------------------- 1 | {application, 2 | dinerl, 3 | [{description, "dinerl"}, 4 | {vsn, git}, 5 | {modules, []}, 6 | {registered, []}, 7 | {env, 8 | [{increment_stat_callback, {dinerl_util, noop}}, 9 | {histogram_stat_callback, {dinerl_util, noop}}, 10 | {max_connections, 10000}]}, 11 | {licenses, ["MIT"]}, 12 | {applications, 13 | [kernel, stdlib, crypto, inets, asn1, public_key, ssl, xmerl, erliam, lhttpc]}]}. 14 | -------------------------------------------------------------------------------- /src/dinerl.erl: -------------------------------------------------------------------------------- 1 | -module(dinerl). 2 | 3 | -define(DINERL_DATA, dinerl_data). 4 | -define(ARGS_KEY, args). 5 | -define(NONE, <<"NONE">>). 6 | -define(ALL_OLD, <<"ALL_OLD">>). 7 | -define(UPDATED_OLD, <<"UPDATED_OLD">>). 8 | -define(ALL_NEW, <<"ALL_NEW">>). 9 | -define(UPDATED_NEW, <<"UPDATED_NEW">>). 10 | 11 | -type access_key_id() :: string(). 12 | -type secret_access_key() :: string(). 13 | -type zone() :: string(). 14 | -type rfcdate() :: string(). 15 | -type field() :: {binary(), binary() | string()}. 16 | -type keyschema_element() :: {binary(), [field()]}. 17 | -type keyschema() :: [keyschema_element()]. 18 | -type jsonf() :: any(). 19 | -type clientarguments() :: {awsv4:credentials(), zone(), rfcdate()}. 20 | -type method() :: 21 | batch_get_item | 22 | get_item | 23 | put_item | 24 | delete_item | 25 | update_item | 26 | create_table | 27 | list_tables | 28 | describe_table | 29 | update_table | 30 | delete_table | 31 | q | 32 | scan | 33 | query_item_20111205 | 34 | query_item_20120810. 35 | -type result() :: 36 | {ok, any()} | 37 | {error, string(), string()} | 38 | {error, term(), timeout | string()} | 39 | {error, atom(), any()}. 40 | 41 | -export_type([access_key_id/0, clientarguments/0, jsonf/0, keyschema/0, method/0, 42 | result/0, secret_access_key/0, zone/0]). 43 | 44 | -export([setup/3, setup/1, setup/0, api/1, api/2, api/3, api/4]). 45 | -export([create_table/4, create_table/5, delete_table/1, delete_table/2]). 46 | -export([describe_table/1, describe_table/2, update_table/3, update_table/4]). 47 | -export([list_tables/0, list_tables/1, list_tables/2, put_item/3, put_item/4]). 48 | -export([delete_item/3, delete_item/4]). 49 | -export([get_item/3, get_item/4, get_item/5]). 50 | -export([get_items/1, get_items/2, get_items/3, get_items/4, get_items/5]). 51 | -export([update_item/3, update_item/4]). 52 | -export([update_item_with_expression/3]). 53 | -export([update_item_with_expression/4]). 54 | -export([update_item_with_expression/5]). 55 | -export([update_item_with_expression/6]). 56 | -export([update_item_with_expression/7]). 57 | -export([query_item/3, query_item/4, query_item/5]). 58 | -export([query/2, query/3, query/4]). 59 | -export([update_data/1]). 60 | -export([batch_write_item/3]). 61 | 62 | -spec setup(access_key_id(), secret_access_key(), zone()) -> {ok, clientarguments()}. 63 | setup(AccessKeyId, SecretAccessKey, Zone) -> 64 | application:set_env(erliam, aws_access_key, AccessKeyId), 65 | application:set_env(erliam, aws_secret_key, SecretAccessKey), 66 | setup_(Zone). 67 | 68 | -spec setup(zone()) -> {ok, clientarguments()}. 69 | setup(Zone) -> 70 | setup_(Zone). 71 | 72 | -spec setup() -> {ok, clientarguments()}. 73 | setup() -> 74 | {ok, Zone} = imds:zone(), 75 | setup_(Zone). 76 | 77 | -spec setup_(string()) -> {ok, clientarguments()}. 78 | setup_(Zone) -> 79 | ets:new(?DINERL_DATA, [named_table, public]), 80 | R = update_data(Zone), 81 | timer:apply_interval(1000, ?MODULE, update_data, [Zone]), 82 | R. 83 | 84 | -spec api(method()) -> result(). 85 | api(Name) -> 86 | api(Name, {struct, []}). 87 | 88 | -spec api(method(), any()) -> result(). 89 | api(Name, Body) -> 90 | api(Name, Body, undefined). 91 | 92 | -spec api(method(), any(), undefined | integer()) -> result(). 93 | api(Name, Body, Timeout) -> 94 | api(Name, Body, Timeout, undefined). 95 | 96 | -spec api(method(), any(), undefined | integer(), undefined | zone()) -> result(). 97 | api(Name, Body, Timeout, Region) -> 98 | try ets:lookup_element(?DINERL_DATA, ?ARGS_KEY, 2) of 99 | {Credentials, Zone, Date} -> 100 | TargetRegion = 101 | case Region of 102 | undefined -> 103 | Zone; 104 | _ -> 105 | Region 106 | end, 107 | dinerl_client:api(Credentials, TargetRegion, Date, Name, Body, Timeout) 108 | catch 109 | _:{badarg, _} -> 110 | {error, missing_credentials, ""} 111 | end. 112 | 113 | -spec create_table(string() | binary(), keyschema(), integer(), integer()) -> jsonf(). 114 | create_table(Name, Key, ReadsPerSecond, WritesPerSecond) -> 115 | create_table(Name, Key, ReadsPerSecond, WritesPerSecond, undefined). 116 | 117 | -spec create_table(string() | binary(), 118 | keyschema(), 119 | integer(), 120 | integer(), 121 | undefined | integer()) -> 122 | jsonf(). 123 | create_table(Name, Key, ReadsPerSecond, WritesPerSecond, Timeout) -> 124 | api(create_table, 125 | [{<<"TableName">>, Name}, 126 | {<<"KeySchema">>, Key}, 127 | {<<"ProvisionedThroughput">>, 128 | [{<<"ReadsPerSecond">>, ReadsPerSecond}, {<<"WritesPerSecond">>, WritesPerSecond}]}], 129 | Timeout). 130 | 131 | delete_table(Name) -> 132 | describe_table(Name, undefined). 133 | 134 | delete_table(Name, Timeout) -> 135 | api(delete_table, [{<<"TableName">>, Name}], Timeout). 136 | 137 | describe_table(Name) -> 138 | describe_table(Name, undefined). 139 | 140 | describe_table(Name, Timeout) -> 141 | api(describe_table, [{<<"TableName">>, Name}], Timeout). 142 | 143 | update_table(Name, ReadsPerSecond, WritesPerSecond) -> 144 | update_table(Name, ReadsPerSecond, WritesPerSecond, undefined). 145 | 146 | update_table(Name, ReadsPerSecond, WritesPerSecond, Timeout) -> 147 | api(update_table, 148 | [{<<"TableName">>, Name}, 149 | {<<"ProvisionedThroughput">>, 150 | [{<<"ReadsPerSecond">>, ReadsPerSecond}, {<<"WritesPerSecond">>, WritesPerSecond}]}], 151 | Timeout). 152 | 153 | list_tables() -> 154 | list_tables([]). 155 | 156 | list_tables(List) -> 157 | list_tables(List, undefined). 158 | 159 | list_tables(List, Timeout) -> 160 | list_tables(List, [], Timeout). 161 | 162 | list_tables([], [], Timeout) -> 163 | list_tables([], {}, Timeout); 164 | list_tables([], Body, Timeout) -> 165 | api(list_tables, Body, Timeout); 166 | list_tables([{start_name, Name} | Rest], Acc, Timeout) -> 167 | list_tables(Rest, [{<<"ExclusiveStartTableName">>, Name} | Acc], Timeout); 168 | list_tables([{limit, N} | Rest], Acc, Timeout) -> 169 | list_tables(Rest, [{<<"Limit">>, N} | Acc], Timeout). 170 | 171 | put_item(Table, Attributes, Options) -> 172 | put_item(Table, Attributes, Options, undefined). 173 | 174 | put_item(Table, Attributes, Options, Timeout) -> 175 | put_item(Table, Attributes, Options, [], Timeout). 176 | 177 | put_item(Table, Attributes, [], PartialBody, Timeout) -> 178 | api(put_item, 179 | [{<<"TableName">>, Table}, {<<"Item">>, Attributes} | PartialBody], 180 | Timeout); 181 | put_item(T, A, [{return, all_old} | Rest], Acc, Timeout) -> 182 | put_item(T, A, Rest, [{<<"ReturnValues">>, ?ALL_OLD} | Acc], Timeout); 183 | put_item(T, A, [{return, none} | Rest], Acc, Timeout) -> 184 | put_item(T, A, Rest, [{<<"ReturnValues">>, ?NONE} | Acc], Timeout); 185 | put_item(T, A, [{expected, V} | Rest], Acc, Timeout) -> 186 | put_item(T, A, Rest, [{<<"Expected">>, attr_updates(V, [])} | Acc], Timeout). 187 | 188 | delete_item(Table, Key, Options) -> 189 | delete_item(Table, Key, Options, undefined). 190 | 191 | delete_item(Table, Key, Options, Timeout) -> 192 | delete_item(Table, Key, Options, [], Timeout). 193 | 194 | delete_item(Table, Key, [], PartialBody, Timeout) -> 195 | api(delete_item, [{<<"TableName">>, Table}, {<<"Key">>, Key} | PartialBody], Timeout); 196 | delete_item(T, K, [{return, all_old} | Rest], Acc, Timeout) -> 197 | delete_item(T, K, Rest, [{<<"ReturnValues">>, ?ALL_OLD} | Acc], Timeout); 198 | delete_item(T, K, [{return, none} | Rest], Acc, Timeout) -> 199 | delete_item(T, K, Rest, [{<<"ReturnValues">>, ?NONE} | Acc], Timeout); 200 | delete_item(T, K, [{expected, V} | Rest], Acc, Timeout) -> 201 | delete_item(T, K, Rest, [{<<"Expected">>, attr_updates(V, [])} | Acc], Timeout). 202 | 203 | get_item(Table, Key, Options) -> 204 | get_item(Table, Key, Options, [], undefined, undefined). 205 | 206 | get_item(Table, Key, Options, Timeout) -> 207 | get_item(Table, Key, Options, [], Timeout, undefined). 208 | 209 | get_item(Table, Key, Options, Timeout, Region) -> 210 | get_item(Table, Key, Options, [], Timeout, Region). 211 | 212 | get_item(T, K, [], Acc, Timeout, Region) -> 213 | api(get_item, [{<<"TableName">>, T}, {<<"Key">>, K} | Acc], Timeout, Region); 214 | get_item(T, K, [{consistent, V} | Rest], Acc, Timeout, Region) -> 215 | get_item(T, K, Rest, [{<<"ConsistentRead">>, V} | Acc], Timeout, Region); 216 | get_item(T, K, [{attrs, V} | Rest], Acc, Timeout, Region) -> 217 | get_item(T, K, Rest, [{<<"AttributesToGet">>, V} | Acc], Timeout, Region). 218 | 219 | get_items(Table, Keys, Options) -> 220 | do_get_items([{Table, Keys, Options}], [], undefined, undefined). 221 | 222 | get_items(Table, Keys, Options, Timeout) -> 223 | do_get_items([{Table, Keys, Options}], [], Timeout, undefined). 224 | 225 | get_items(Table, Keys, Options, Timeout, Region) -> 226 | do_get_items([{Table, Keys, Options}], [], Timeout, Region). 227 | 228 | get_items(MultiTableQuery) -> 229 | do_get_items(MultiTableQuery, [], undefined, undefined). 230 | 231 | get_items(MultiTableQuery, Timeout) -> 232 | do_get_items(MultiTableQuery, [], Timeout, undefined). 233 | 234 | do_get_items([], Acc, Timeout, Region) -> 235 | api(batch_get_item, [{<<"RequestItems">>, Acc}], Timeout, Region); 236 | do_get_items([{Table, Keys, Options} | Rest], Acc, Timeout, Region) -> 237 | Attrs = proplists:get_value(attrs, Options, []), 238 | do_get_items(Rest, [get_body_request(Table, Keys, Attrs) | Acc], Timeout, Region). 239 | 240 | get_body_request(Table, Keys, []) -> 241 | {Table, [{<<"Keys">>, Keys}]}; 242 | get_body_request(Table, Keys, Attrs) -> 243 | {Table, [{<<"Keys">>, Keys}, {<<"AttributesToGet">>, Attrs}]}. 244 | 245 | update_item_with_expression(TableName, Key, UpdateExpression) -> 246 | update_item_with_expression(TableName, 247 | Key, 248 | UpdateExpression, 249 | undefined, 250 | <<"NONE">>, 251 | [], 252 | undefined). 253 | 254 | update_item_with_expression(TableName, 255 | Key, 256 | UpdateExpression, 257 | ExpressionAttributeValues) -> 258 | update_item_with_expression(TableName, 259 | Key, 260 | UpdateExpression, 261 | ExpressionAttributeValues, 262 | <<"NONE">>, 263 | [], 264 | undefined). 265 | 266 | update_item_with_expression(TableName, 267 | Key, 268 | UpdateExpression, 269 | ExpressionAttributeValues, 270 | ReturnValues) -> 271 | update_item_with_expression(TableName, 272 | Key, 273 | UpdateExpression, 274 | ExpressionAttributeValues, 275 | ReturnValues, 276 | [], 277 | undefined). 278 | 279 | update_item_with_expression(TableName, 280 | Key, 281 | UpdateExpression, 282 | ExpressionAttributeValues, 283 | ReturnValues, 284 | Acc) -> 285 | update_item_with_expression(TableName, 286 | Key, 287 | UpdateExpression, 288 | ExpressionAttributeValues, 289 | ReturnValues, 290 | Acc, 291 | undefined). 292 | 293 | update_item_with_expression(TableName, 294 | Key, 295 | UpdateExpression, 296 | ExpressionAttributeValues, 297 | ReturnValues, 298 | Acc, 299 | Timeout) -> 300 | MandatoryParams = 301 | [{<<"TableName">>, TableName}, 302 | {<<"Key">>, Key}, 303 | {<<"UpdateExpression">>, UpdateExpression}], 304 | OptionalParams = 305 | [{<<"ExpressionAttributeValues">>, ExpressionAttributeValues}, 306 | {<<"ReturnValues">>, ReturnValues}], 307 | DefinedOptionalParams = 308 | lists:filter(fun ({_, undefined}) -> 309 | false; 310 | (_) -> 311 | true 312 | end, 313 | OptionalParams), 314 | 315 | api(update_item, MandatoryParams ++ DefinedOptionalParams ++ Acc, Timeout). 316 | 317 | update_item(Table, Key, Options) -> 318 | update_item(Table, Key, Options, undefined). 319 | 320 | update_item(Table, Key, Options, Timeout) -> 321 | update_item(Table, Key, Options, [], Timeout). 322 | 323 | update_item(T, K, [], Acc, Timeout) -> 324 | api(update_item, [{<<"TableName">>, T}, {<<"Key">>, K} | Acc], Timeout); 325 | update_item(T, K, [{update, AttributeUpdates} | Rest], Acc, Timeout) -> 326 | update_item(T, 327 | K, 328 | Rest, 329 | [{<<"AttributeUpdates">>, attr_updates(AttributeUpdates, [])} | Acc], 330 | Timeout); 331 | update_item(T, K, [{expected, V} | Rest], Acc, Timeout) -> 332 | update_item(T, K, Rest, [{<<"Expected">>, attr_updates(V, [])} | Acc], Timeout); 333 | update_item(T, K, [{return, none} | Rest], Acc, Timeout) -> 334 | update_item(T, K, Rest, [{<<"ReturnValues">>, ?NONE} | Acc], Timeout); 335 | update_item(T, K, [{return, all_old} | Rest], Acc, Timeout) -> 336 | update_item(T, K, Rest, [{<<"ReturnValues">>, ?ALL_OLD} | Acc], Timeout); 337 | update_item(T, K, [{return, updated_old} | Rest], Acc, Timeout) -> 338 | update_item(T, K, Rest, [{<<"ReturnValues">>, ?UPDATED_OLD} | Acc], Timeout); 339 | update_item(T, K, [{return, all_new} | Rest], Acc, Timeout) -> 340 | update_item(T, K, Rest, [{<<"ReturnValues">>, ?ALL_NEW} | Acc], Timeout); 341 | update_item(T, K, [{return, updated_new} | Rest], Acc, Timeout) -> 342 | update_item(T, K, Rest, [{<<"ReturnValues">>, ?UPDATED_NEW} | Acc], Timeout). 343 | 344 | %% query_item options: 345 | %% Uses amazon api version: 346 | %% http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/API_Query_v20111205.html 347 | %% limit: int, max number of results 348 | %% count: bool, only return the total count 349 | %% scan_index_forward: bool, set to false to reverse the sort order 350 | %% consistent: bool, make a consistent read, default false 351 | %% exclusive_start_key: output from LastEvaluatedKey when limit(size or limit param) is reached 352 | %% attrs: array( binary ), [<<"a">>,<<"b">>], only return these attributes 353 | %% range_condition: {array( attributes), operation } 354 | %% eg, dinerl:query_item(<<"table">>,[{<<"S">>, <<"hash_value">>}], 355 | %% [{range_condition, { [[{<<"S">>, <<"range_value">>}]] ,<<"EQ">> }}]). 356 | query_item(Table, Key, Options) -> 357 | query_item(Table, Key, Options, [], undefined, undefined). 358 | 359 | query_item(Table, Key, Options, Timeout) -> 360 | query_item(Table, Key, Options, [], Timeout, undefined). 361 | 362 | query_item(Table, Key, Options, Timeout, Region) -> 363 | query_item(Table, Key, Options, [], Timeout, Region). 364 | 365 | query_item(T, K, List, Acc, Timeout, Region) -> 366 | NewParameters = 367 | [{<<"TableName">>, T}, {<<"HashKeyValue">>, K} | convert_query_parameters(List, Acc)], 368 | api(query_item_20111205, NewParameters, Timeout, Region). 369 | 370 | %% Uses new API 371 | query(Table, Options) -> 372 | query(Table, Options, undefined, undefined). 373 | 374 | query(Table, Options, Timeout) -> 375 | query(Table, Options, Timeout, undefined). 376 | 377 | query(Table, Options, Timeout, Region) -> 378 | NewParameters = [{<<"TableName">>, Table} | convert_query_parameters(Options, [])], 379 | api(query_item_20120810, NewParameters, Timeout, Region). 380 | 381 | convert_query_parameters([], Acc) -> 382 | Acc; 383 | convert_query_parameters([{limit, V} | Rest], Acc) -> 384 | convert_query_parameters(Rest, [{<<"Limit">>, V} | Acc]); 385 | convert_query_parameters([{count, V} | Rest], Acc) -> 386 | convert_query_parameters(Rest, [{<<"Count">>, V} | Acc]); 387 | convert_query_parameters([{index_name, IndexName} | Rest], Acc) -> 388 | convert_query_parameters(Rest, [{<<"IndexName">>, IndexName} | Acc]); 389 | convert_query_parameters([{scan_index_forward, V} | Rest], Acc) -> 390 | convert_query_parameters(Rest, [{<<"ScanIndexForward">>, V} | Acc]); 391 | convert_query_parameters([{consistent, V} | Rest], Acc) -> 392 | convert_query_parameters(Rest, [{<<"ConsistentRead">>, V} | Acc]); 393 | convert_query_parameters([{exclusive_start_key, V} | Rest], Acc) -> 394 | convert_query_parameters(Rest, [{<<"ExclusiveStartKey">>, V} | Acc]); 395 | convert_query_parameters([{range_condition, {V, Op}} | Rest], Acc) -> 396 | convert_query_parameters(Rest, 397 | [{<<"RangeKeyCondition">>, 398 | [{<<"AttributeValueList">>, V}, {<<"ComparisonOperator">>, Op}]} 399 | | Acc]); 400 | convert_query_parameters([{attrs, V} | Rest], Acc) -> 401 | convert_query_parameters(Rest, [{<<"AttributesToGet">>, V} | Acc]); 402 | convert_query_parameters([{project_expression, ProjectionExpression} | Rest], Acc) -> 403 | convert_query_parameters(Rest, 404 | [{<<"ProjectionExpression">>, ProjectionExpression} | Acc]); 405 | convert_query_parameters([{key_condition_expression, KeyConditionExpression} | Rest], 406 | Acc) -> 407 | convert_query_parameters(Rest, 408 | [{<<"KeyConditionExpression">>, KeyConditionExpression} | Acc]); 409 | convert_query_parameters([{condition_expression, KeyConditionExpression} | Rest], Acc) -> 410 | convert_query_parameters(Rest, 411 | [{<<"ConditionExpression">>, KeyConditionExpression} | Acc]); 412 | convert_query_parameters([{filter_expression, KeyConditionExpression} | Rest], Acc) -> 413 | convert_query_parameters(Rest, [{<<"FilterExpression">>, KeyConditionExpression} | Acc]); 414 | convert_query_parameters([{expression_attribute_values, ExpressionAttributeValues} 415 | | Rest], 416 | Acc) -> 417 | convert_query_parameters(Rest, 418 | [{<<"ExpressionAttributeValues">>, ExpressionAttributeValues} | Acc]); 419 | convert_query_parameters([{expression_attribute_names, ExpressionAttributeNames} | Rest], 420 | Acc) -> 421 | convert_query_parameters(Rest, 422 | [{<<"ExpressionAttributeNames">>, ExpressionAttributeNames} | Acc]). 423 | 424 | batch_write_item(TableName, PutItems, DeleteKeys) -> 425 | api(batch_write_item, 426 | [{<<"RequestItems">>, 427 | [{TableName, 428 | lists:map(fun make_batch_put/1, PutItems) 429 | ++ lists:map(fun make_batch_delete/1, DeleteKeys)}]}]). 430 | 431 | make_batch_put(Item) -> 432 | [{<<"PutRequest">>, [{<<"Item">>, Item}]}]. 433 | 434 | make_batch_delete(Key) -> 435 | [{<<"DeleteRequest">>, [{<<"Key">>, Key}]}]. 436 | 437 | %% Internal 438 | %% 439 | %% Every second it updates the Date part of the arguments and copies the latest cached 440 | %% credentials from erliam. 441 | -spec update_data(zone()) -> {ok, clientarguments()}. 442 | update_data(Zone) -> 443 | NewDate = awsv4:isonow(), 444 | NewArgs = {erliam:credentials(), Zone, NewDate}, 445 | ets:insert(?DINERL_DATA, {?ARGS_KEY, NewArgs}), 446 | {ok, NewArgs}. 447 | 448 | expected([], Acc) -> 449 | Acc; 450 | expected([{Option, Value} | Rest], Acc) -> 451 | expected(Rest, [value_and_action({Option, Value}) | Acc]). 452 | 453 | attr_updates([], Acc) -> 454 | Acc; 455 | attr_updates([{AttrName, Opts} | Rest], Acc) -> 456 | attr_updates(Rest, [{AttrName, expected(Opts, [])} | Acc]). 457 | 458 | value_and_action({value, V}) -> 459 | {<<"Value">>, V}; 460 | value_and_action({action, put}) -> 461 | {<<"Action">>, <<"PUT">>}; 462 | value_and_action({action, add}) -> 463 | {<<"Action">>, <<"ADD">>}; 464 | value_and_action({action, delete}) -> 465 | {<<"Action">>, <<"DELETE">>}; 466 | value_and_action({exists, V}) -> 467 | {<<"Exists">>, V}. 468 | -------------------------------------------------------------------------------- /src/dinerl_client.erl: -------------------------------------------------------------------------------- 1 | -module(dinerl_client). 2 | 3 | -export([api/5, api/6]). 4 | 5 | %% 6 | %% Item related operations 7 | %% 8 | -spec method_name(dinerl:method()) -> string(). 9 | method_name(batch_get_item) -> 10 | "DynamoDB_20111205.BatchGetItem"; 11 | method_name(batch_write_item) -> 12 | "DynamoDB_20120810.BatchWriteItem"; 13 | method_name(get_item) -> 14 | "DynamoDB_20111205.GetItem"; 15 | method_name(put_item) -> 16 | "DynamoDB_20111205.PutItem"; 17 | method_name(delete_item) -> 18 | "DynamoDB_20111205.DeleteItem"; 19 | method_name(update_item) -> 20 | "DynamoDB_20120810.UpdateItem"; 21 | %% 22 | %% Table related operations 23 | %% 24 | method_name(create_table) -> 25 | "DynamoDB_20111205.CreateTable"; 26 | method_name(list_tables) -> 27 | "DynamoDB_20111205.ListTables"; 28 | method_name(describe_table) -> 29 | "DynamoDB_20111205.DescribeTable"; 30 | method_name(update_table) -> 31 | "DynamoDB_20111205.UpdateTable"; 32 | method_name(delete_table) -> 33 | "DynamoDB_20111205.DeleteTable"; 34 | %% 35 | %% query interface 36 | %% 37 | method_name(query_item_20120810) -> 38 | "DynamoDB_20120810.Query"; 39 | method_name(query_item_20111205) -> 40 | "DynamoDB_20111205.Query"; 41 | method_name(scan) -> 42 | "DynamoDB_20111205.Scan". 43 | 44 | -spec api(awsv4:credentials(), 45 | dinerl:zone(), 46 | dynamodb:aws_datetime(), 47 | dinerl:method(), 48 | any(), 49 | undefined | integer()) -> 50 | dinerl:result(). 51 | api(Credentials, Zone, ISODate, Name, Body, Timeout) -> 52 | case dynamodb:call(Credentials, 53 | Zone, 54 | method_name(Name), 55 | ISODate, 56 | dmochijson2:encode(Body), 57 | Timeout) 58 | of 59 | {ok, Response} -> 60 | {ok, dmochijson2:decode(Response)}; 61 | {error, Code, Reason} -> 62 | {error, Code, Reason} 63 | end. 64 | 65 | api(Credentials, Zone, ISODate, Name, Body) -> 66 | api(Credentials, Zone, ISODate, Name, Body, undefined). 67 | -------------------------------------------------------------------------------- /src/dinerl_util.erl: -------------------------------------------------------------------------------- 1 | -module(dinerl_util). 2 | 3 | %% Because noop/2 is used as a stat_callback 4 | -hank([{unnecessary_function_arguments, [noop]}]). 5 | 6 | -export([get_env/1, noop/2, time_call/2, time_call/3, increment/1, increment/2]). 7 | 8 | %%%=================================================================== 9 | %%% API 10 | %%%=================================================================== 11 | 12 | %% @equiv time_call(Metric, Fun, 1.0) 13 | -spec time_call(term(), fun(() -> Result)) -> Result. 14 | time_call(Metric, Fun) -> 15 | time_call(Metric, Fun, 1.0). 16 | 17 | %% @doc Evaluates Fun() and (with a probability of Ratio) reports its evaluation 18 | %% time as a histogram metric called Metric. 19 | -spec time_call(term(), fun(() -> Result), number()) -> Result. 20 | time_call(Metric, Fun, 1.0) -> 21 | Start = erlang:system_time(microsecond), 22 | Result = Fun(), 23 | End = erlang:system_time(microsecond), 24 | Diff = End - Start, 25 | histogram(Metric, Diff), 26 | Result; 27 | time_call(Metric, Fun, Ratio) when is_number(Ratio) -> 28 | Start = erlang:system_time(microsecond), 29 | Result = Fun(), 30 | case rand:uniform() =< Ratio of 31 | true -> 32 | End = erlang:system_time(microsecond), 33 | Diff = End - Start, 34 | histogram(Metric, Diff); 35 | false -> 36 | ok 37 | end, 38 | Result. 39 | 40 | histogram(Metric, Value) -> 41 | {StatMod, StatFun} = histogram_stat_callback(), 42 | erlang:apply(StatMod, StatFun, [Metric, Value]). 43 | 44 | increment(Metric) -> 45 | increment(Metric, 1). 46 | 47 | increment(Metric, Value) -> 48 | {StatMod, StatFun} = increment_stat_callback(), 49 | erlang:apply(StatMod, StatFun, [Metric, Value]). 50 | 51 | noop(_Key, _Value) -> 52 | ok. 53 | 54 | get_env(Key) -> 55 | case application:get_env(dinerl, Key) of 56 | {ok, Value} -> 57 | Value; 58 | undefined -> 59 | exit({undefined_configuration, Key}) 60 | end. 61 | 62 | %%%=================================================================== 63 | %%% Internal Functions 64 | %%%=================================================================== 65 | histogram_stat_callback() -> 66 | get_env(histogram_stat_callback). 67 | 68 | increment_stat_callback() -> 69 | get_env(increment_stat_callback). 70 | -------------------------------------------------------------------------------- /src/dmochijson2.erl: -------------------------------------------------------------------------------- 1 | %% @author Bob Ippolito 2 | %% @copyright 2007 Mochi Media, Inc. 3 | 4 | %% @doc Yet another JSON (RFC 4627) library for Erlang. mochijson2 works 5 | %% with binaries as strings, arrays as lists (without an {array, _}) 6 | %% wrapper and it only knows how to decode UTF-8 (and ASCII). 7 | %% 8 | %% JSON terms are decoded as follows (javascript -> erlang): 9 | %% 16 | %% 22 | %% The encoder will accept the same format that the decoder will produce, 23 | %% but will also allow additional cases for leniency: 24 | %% 39 | 40 | -module(dmochijson2). 41 | 42 | -export([encoder/1, encode/1]). 43 | -export([decoder/1, decode/1, decode/2]). 44 | 45 | %% It was the style used by mochiweb 46 | -elvis([{elvis_style, param_pattern_matching, #{side => left}}]). 47 | 48 | %% This is a macro to placate syntax highlighters.. 49 | -define(Q, $\"). 50 | -define(ADV_COL(S, N), 51 | S#decoder{offset = N + S#decoder.offset, column = N + S#decoder.column}). 52 | -define(INC_COL(S), 53 | S#decoder{offset = 1 + S#decoder.offset, column = 1 + S#decoder.column}). 54 | -define(INC_CHAR(S, C), 55 | case C of 56 | $\n -> 57 | S#decoder{column = 1, 58 | line = 1 + S#decoder.line, 59 | offset = 1 + S#decoder.offset}; 60 | _ -> 61 | S#decoder{column = 1 + S#decoder.column, offset = 1 + S#decoder.offset} 62 | end). 63 | -define(IS_WHITESPACE(C), C =:= $\s orelse C =:= $\t orelse C =:= $\r orelse C =:= $\n). 64 | 65 | -type json_string() :: atom | binary(). 66 | -type json_number() :: integer() | float(). 67 | -type json_array() :: [json_term()]. 68 | -type json_object() :: {struct, [{json_string(), json_term()}]}. 69 | -type json_eep18_object() :: {[{json_string(), json_term()}]}. 70 | -type json_iodata() :: {json, iodata()}. 71 | -type json_term() :: 72 | json_string() | 73 | json_number() | 74 | json_array() | 75 | json_object() | 76 | json_eep18_object() | 77 | json_iodata(). 78 | 79 | -export_type([json_term/0]). 80 | 81 | -record(encoder, {handler = null, utf8 = false}). 82 | -record(decoder, {object_hook = null, offset = 0, line = 1, column = 1, state = null}). 83 | 84 | %% @spec encoder([encoder_option()]) -> function() 85 | %% @doc Create an encoder/1 with the given options. 86 | %% @type encoder_option() = handler_option() | utf8_option() 87 | %% @type utf8_option() = boolean(). Emit unicode as utf8 (default - false) 88 | encoder(Options) -> 89 | State = parse_encoder_options(Options, #encoder{}), 90 | fun(O) -> json_encode(O, State) end. 91 | 92 | %% @spec encode(json_term()) -> iodata() 93 | %% @doc Encode the given as JSON to an iodata. 94 | -spec encode(json_term()) -> iodata(). 95 | encode(Any) -> 96 | json_encode(Any, #encoder{}). 97 | 98 | %% @spec decoder([decoder_option()]) -> function() 99 | %% @doc Create a decoder/1 with the given options. 100 | decoder(Options) -> 101 | State = parse_decoder_options(Options, #decoder{}), 102 | fun(O) -> json_decode(O, State) end. 103 | 104 | %% @spec decode(iodata(), [{format, proplist | eep18 | struct}]) -> json_term() 105 | %% @doc Decode the given iodata to Erlang terms using the given object format 106 | %% for decoding, where proplist returns JSON objects as [{binary(), json_term()}] 107 | %% proplists, eep18 returns JSON objects as {[binary(), json_term()]}, and struct 108 | %% returns them as-is. 109 | decode(S, Options) -> 110 | json_decode(S, parse_decoder_options(Options, #decoder{})). 111 | 112 | %% @spec decode(iodata()) -> json_term() 113 | %% @doc Decode the given iodata to Erlang terms. 114 | decode(S) -> 115 | json_decode(S, #decoder{}). 116 | 117 | %% Internal API 118 | 119 | parse_encoder_options([], State) -> 120 | State; 121 | parse_encoder_options([{handler, Handler} | Rest], State) -> 122 | parse_encoder_options(Rest, State#encoder{handler = Handler}); 123 | parse_encoder_options([{utf8, Switch} | Rest], State) -> 124 | parse_encoder_options(Rest, State#encoder{utf8 = Switch}). 125 | 126 | parse_decoder_options([], State) -> 127 | State; 128 | parse_decoder_options([{object_hook, Hook} | Rest], State) -> 129 | parse_decoder_options(Rest, State#decoder{object_hook = Hook}); 130 | parse_decoder_options([{format, Format} | Rest], State) 131 | when Format =:= struct orelse Format =:= eep18 orelse Format =:= proplist -> 132 | parse_decoder_options(Rest, State#decoder{object_hook = Format}). 133 | 134 | json_encode(true, _State) -> 135 | <<"true">>; 136 | json_encode(false, _State) -> 137 | <<"false">>; 138 | json_encode(null, _State) -> 139 | <<"null">>; 140 | json_encode(I, _State) when is_integer(I) -> 141 | integer_to_list(I); 142 | json_encode(F, _State) when is_float(F) -> 143 | dmochinum:digits(F); 144 | json_encode(S, State) when is_binary(S); is_atom(S) -> 145 | json_encode_string(S, State); 146 | json_encode(Props = [{K, _} | _], State) 147 | when K =/= struct andalso K =/= array andalso K =/= json -> 148 | json_encode_proplist(Props, State); 149 | json_encode({struct, Props}, State) when is_list(Props) -> 150 | json_encode_proplist(Props, State); 151 | json_encode({Props}, State) when is_list(Props) -> 152 | json_encode_proplist(Props, State); 153 | json_encode({}, State) -> 154 | json_encode_proplist([], State); 155 | json_encode(Array, State) when is_list(Array) -> 156 | json_encode_array(Array, State); 157 | json_encode({array, Array}, State) when is_list(Array) -> 158 | json_encode_array(Array, State); 159 | json_encode({json, IoList}, _State) -> 160 | IoList; 161 | json_encode(Bad, #encoder{handler = null}) -> 162 | exit({json_encode, {bad_term, Bad}}); 163 | json_encode(Bad, State = #encoder{handler = Handler}) -> 164 | json_encode(Handler(Bad), State). 165 | 166 | json_encode_array([], _State) -> 167 | <<"[]">>; 168 | json_encode_array(L, State) -> 169 | F = fun(O, Acc) -> [$,, json_encode(O, State) | Acc] end, 170 | [$, | Acc1] = lists:foldl(F, "[", L), 171 | lists:reverse([$\] | Acc1]). 172 | 173 | json_encode_proplist([], _State) -> 174 | <<"{}">>; 175 | json_encode_proplist(Props, State) -> 176 | F = fun({K, V}, Acc) -> 177 | KS = json_encode_string(K, State), 178 | VS = json_encode(V, State), 179 | [$,, VS, $:, KS | Acc] 180 | end, 181 | [$, | Acc1] = lists:foldl(F, "{", Props), 182 | lists:reverse([$\} | Acc1]). 183 | 184 | json_encode_string(A, State) when is_atom(A) -> 185 | L = atom_to_list(A), 186 | case json_string_is_safe(L) of 187 | true -> 188 | [?Q, L, ?Q]; 189 | false -> 190 | json_encode_string_unicode(xmerl_ucs:from_utf8(L), State, [?Q]) 191 | end; 192 | json_encode_string(B, State) when is_binary(B) -> 193 | case json_bin_is_safe(B) of 194 | true -> 195 | [?Q, B, ?Q]; 196 | false -> 197 | json_encode_string_unicode(xmerl_ucs:from_utf8(B), State, [?Q]) 198 | end; 199 | json_encode_string(I, _State) when is_integer(I) -> 200 | [?Q, integer_to_list(I), ?Q]; 201 | json_encode_string(L, State) when is_list(L) -> 202 | case json_string_is_safe(L) of 203 | true -> 204 | [?Q, L, ?Q]; 205 | false -> 206 | json_encode_string_unicode(L, State, [?Q]) 207 | end. 208 | 209 | json_string_is_safe([]) -> 210 | true; 211 | json_string_is_safe([C | Rest]) -> 212 | case C of 213 | ?Q -> 214 | false; 215 | $\\ -> 216 | false; 217 | $\b -> 218 | false; 219 | $\f -> 220 | false; 221 | $\n -> 222 | false; 223 | $\r -> 224 | false; 225 | $\t -> 226 | false; 227 | C when C >= 0, C < $\s; C >= 16#7f, C =< 16#10FFFF -> 228 | false; 229 | C when C < 16#7f -> 230 | json_string_is_safe(Rest); 231 | _ -> 232 | false 233 | end. 234 | 235 | json_bin_is_safe(<<>>) -> 236 | true; 237 | json_bin_is_safe(<>) -> 238 | case C of 239 | ?Q -> 240 | false; 241 | $\\ -> 242 | false; 243 | $\b -> 244 | false; 245 | $\f -> 246 | false; 247 | $\n -> 248 | false; 249 | $\r -> 250 | false; 251 | $\t -> 252 | false; 253 | C when C >= 0, C < $\s; C >= 16#7f -> 254 | false; 255 | C when C < 16#7f -> 256 | json_bin_is_safe(Rest) 257 | end. 258 | 259 | json_encode_string_unicode([], _State, Acc) -> 260 | lists:reverse([$\" | Acc]); 261 | json_encode_string_unicode([C | Cs], State, Acc) -> 262 | Acc1 = 263 | case C of 264 | ?Q -> 265 | [?Q, $\\ | Acc]; 266 | %% Escaping solidus is only useful when trying to protect 267 | %% against "" injection attacks which are only 268 | %% possible when JSON is inserted into a HTML document 269 | %% in-line. mochijson2 does not protect you from this, so 270 | %% if you do insert directly into HTML then you need to 271 | %% uncomment the following case or escape the output of encode. 272 | %% 273 | %% $/ -> 274 | %% [$/, $\\ | Acc]; 275 | %% 276 | $\\ -> 277 | [$\\, $\\ | Acc]; 278 | $\b -> 279 | [$b, $\\ | Acc]; 280 | $\f -> 281 | [$f, $\\ | Acc]; 282 | $\n -> 283 | [$n, $\\ | Acc]; 284 | $\r -> 285 | [$r, $\\ | Acc]; 286 | $\t -> 287 | [$t, $\\ | Acc]; 288 | C when C >= 0, C < $\s -> 289 | [unihex(C) | Acc]; 290 | C when C >= 16#7f, C =< 16#10FFFF, State#encoder.utf8 -> 291 | [xmerl_ucs:to_utf8(C) | Acc]; 292 | C when C >= 16#7f, C =< 16#10FFFF, not State#encoder.utf8 -> 293 | [unihex(C) | Acc]; 294 | C when C < 16#7f -> 295 | [C | Acc]; 296 | _ -> 297 | exit({json_encode, {bad_char, C}}) 298 | end, 299 | json_encode_string_unicode(Cs, State, Acc1). 300 | 301 | hexdigit(C) when C >= 0, C =< 9 -> 302 | C + $0; 303 | hexdigit(C) when C =< 15 -> 304 | C + $a - 10. 305 | 306 | unihex(C) when C < 16#10000 -> 307 | <> = <>, 308 | Digits = [hexdigit(D) || D <- [D3, D2, D1, D0]], 309 | [$\\, $u | Digits]; 310 | unihex(C) when C =< 16#10FFFF -> 311 | N = C - 16#10000, 312 | S1 = 16#d800 bor (N bsr 10) band 16#3ff, 313 | S2 = 16#dc00 bor N band 16#3ff, 314 | [unihex(S1), unihex(S2)]. 315 | 316 | json_decode(L, S) when is_list(L) -> 317 | json_decode(iolist_to_binary(L), S); 318 | json_decode(B, S) -> 319 | {Res, S1} = decode1(B, S), 320 | {eof, _} = tokenize(B, S1#decoder{state = trim}), 321 | Res. 322 | 323 | decode1(B, S = #decoder{state = null}) -> 324 | case tokenize(B, S#decoder{state = any}) of 325 | {{const, C}, S1} -> 326 | {C, S1}; 327 | {start_array, S1} -> 328 | decode_array(B, S1); 329 | {start_object, S1} -> 330 | decode_object(B, S1) 331 | end. 332 | 333 | make_object(V, #decoder{object_hook = N}) when N =:= null orelse N =:= struct -> 334 | V; 335 | make_object({struct, P}, #decoder{object_hook = eep18}) -> 336 | {P}; 337 | make_object({struct, P}, #decoder{object_hook = proplist}) -> 338 | P; 339 | make_object(V, #decoder{object_hook = Hook}) -> 340 | Hook(V). 341 | 342 | decode_object(B, S) -> 343 | decode_object(B, S#decoder{state = key}, []). 344 | 345 | decode_object(B, S = #decoder{state = key}, Acc) -> 346 | case tokenize(B, S) of 347 | {end_object, S1} -> 348 | V = make_object({struct, lists:reverse(Acc)}, S1), 349 | {V, S1#decoder{state = null}}; 350 | {{const, K}, S1} -> 351 | {colon, S2} = tokenize(B, S1), 352 | {V, S3} = decode1(B, S2#decoder{state = null}), 353 | decode_object(B, S3#decoder{state = comma}, [{K, V} | Acc]) 354 | end; 355 | decode_object(B, S = #decoder{state = comma}, Acc) -> 356 | case tokenize(B, S) of 357 | {end_object, S1} -> 358 | V = make_object({struct, lists:reverse(Acc)}, S1), 359 | {V, S1#decoder{state = null}}; 360 | {comma, S1} -> 361 | decode_object(B, S1#decoder{state = key}, Acc) 362 | end. 363 | 364 | decode_array(B, S) -> 365 | decode_array(B, S#decoder{state = any}, []). 366 | 367 | decode_array(B, S = #decoder{state = any}, Acc) -> 368 | case tokenize(B, S) of 369 | {end_array, S1} -> 370 | {lists:reverse(Acc), S1#decoder{state = null}}; 371 | {start_array, S1} -> 372 | {Array, S2} = decode_array(B, S1), 373 | decode_array(B, S2#decoder{state = comma}, [Array | Acc]); 374 | {start_object, S1} -> 375 | {Array, S2} = decode_object(B, S1), 376 | decode_array(B, S2#decoder{state = comma}, [Array | Acc]); 377 | {{const, Const}, S1} -> 378 | decode_array(B, S1#decoder{state = comma}, [Const | Acc]) 379 | end; 380 | decode_array(B, S = #decoder{state = comma}, Acc) -> 381 | case tokenize(B, S) of 382 | {end_array, S1} -> 383 | {lists:reverse(Acc), S1#decoder{state = null}}; 384 | {comma, S1} -> 385 | decode_array(B, S1#decoder{state = any}, Acc) 386 | end. 387 | 388 | tokenize_string(B, S = #decoder{offset = O}) -> 389 | case tokenize_string_fast(B, O) of 390 | {escape, O1} -> 391 | Length = O1 - O, 392 | S1 = ?ADV_COL(S, Length), 393 | <<_:O/binary, Head:Length/binary, _/binary>> = B, 394 | tokenize_string(B, S1, lists:reverse(binary_to_list(Head))); 395 | O1 -> 396 | Length = O1 - O, 397 | <<_:O/binary, String:Length/binary, ?Q, _/binary>> = B, 398 | {{const, String}, ?ADV_COL(S, Length + 1)} 399 | end. 400 | 401 | tokenize_string_fast(B, O) -> 402 | case B of 403 | <<_:O/binary, ?Q, _/binary>> -> 404 | O; 405 | <<_:O/binary, $\\, _/binary>> -> 406 | {escape, O}; 407 | <<_:O/binary, C1, _/binary>> when C1 < 128 -> 408 | tokenize_string_fast(B, 1 + O); 409 | <<_:O/binary, C1, C2, _/binary>> 410 | when C1 >= 194 andalso C1 =< 223, C2 >= 128 andalso C2 =< 191 -> 411 | tokenize_string_fast(B, 2 + O); 412 | <<_:O/binary, C1, C2, C3, _/binary>> 413 | when C1 >= 224 andalso C1 =< 239, C2 >= 128 andalso C2 =< 191, 414 | C3 >= 128 andalso C3 =< 191 -> 415 | tokenize_string_fast(B, 3 + O); 416 | <<_:O/binary, C1, C2, C3, C4, _/binary>> 417 | when C1 >= 240 andalso C1 =< 244, C2 >= 128 andalso C2 =< 191, 418 | C3 >= 128 andalso C3 =< 191, C4 >= 128 andalso C4 =< 191 -> 419 | tokenize_string_fast(B, 4 + O); 420 | _ -> 421 | error(invalid_utf8) 422 | end. 423 | 424 | tokenize_string(B, S = #decoder{offset = O}, Acc) -> 425 | case B of 426 | <<_:O/binary, ?Q, _/binary>> -> 427 | {{const, iolist_to_binary(lists:reverse(Acc))}, ?INC_COL(S)}; 428 | <<_:O/binary, "\\\"", _/binary>> -> 429 | tokenize_string(B, ?ADV_COL(S, 2), [$\" | Acc]); 430 | <<_:O/binary, "\\\\", _/binary>> -> 431 | tokenize_string(B, ?ADV_COL(S, 2), [$\\ | Acc]); 432 | <<_:O/binary, "\\/", _/binary>> -> 433 | tokenize_string(B, ?ADV_COL(S, 2), [$/ | Acc]); 434 | <<_:O/binary, "\\b", _/binary>> -> 435 | tokenize_string(B, ?ADV_COL(S, 2), [$\b | Acc]); 436 | <<_:O/binary, "\\f", _/binary>> -> 437 | tokenize_string(B, ?ADV_COL(S, 2), [$\f | Acc]); 438 | <<_:O/binary, "\\n", _/binary>> -> 439 | tokenize_string(B, ?ADV_COL(S, 2), [$\n | Acc]); 440 | <<_:O/binary, "\\r", _/binary>> -> 441 | tokenize_string(B, ?ADV_COL(S, 2), [$\r | Acc]); 442 | <<_:O/binary, "\\t", _/binary>> -> 443 | tokenize_string(B, ?ADV_COL(S, 2), [$\t | Acc]); 444 | <<_:O/binary, "\\u", C3, C2, C1, C0, Rest/binary>> -> 445 | case erlang:list_to_integer([C3, C2, C1, C0], 16) of 446 | C when C > 16#D7FF, C < 16#DC00 -> 447 | %% coalesce UTF-16 surrogate pair 448 | <<"\\u", D3, D2, D1, D0, _/binary>> = Rest, 449 | D = erlang:list_to_integer([D3, D2, D1, D0], 16), 450 | [CodePoint] = 451 | xmerl_ucs:from_utf16be(<>), 453 | Acc1 = 454 | lists:reverse( 455 | xmerl_ucs:to_utf8(CodePoint), Acc), 456 | tokenize_string(B, ?ADV_COL(S, 12), Acc1); 457 | C -> 458 | Acc1 = 459 | lists:reverse( 460 | xmerl_ucs:to_utf8(C), Acc), 461 | tokenize_string(B, ?ADV_COL(S, 6), Acc1) 462 | end; 463 | <<_:O/binary, C1, _/binary>> when C1 < 128 -> 464 | tokenize_string(B, ?INC_CHAR(S, C1), [C1 | Acc]); 465 | <<_:O/binary, C1, C2, _/binary>> 466 | when C1 >= 194 andalso C1 =< 223, C2 >= 128 andalso C2 =< 191 -> 467 | tokenize_string(B, ?ADV_COL(S, 2), [C2, C1 | Acc]); 468 | <<_:O/binary, C1, C2, C3, _/binary>> 469 | when C1 >= 224 andalso C1 =< 239, C2 >= 128 andalso C2 =< 191, 470 | C3 >= 128 andalso C3 =< 191 -> 471 | tokenize_string(B, ?ADV_COL(S, 3), [C3, C2, C1 | Acc]); 472 | <<_:O/binary, C1, C2, C3, C4, _/binary>> 473 | when C1 >= 240 andalso C1 =< 244, C2 >= 128 andalso C2 =< 191, 474 | C3 >= 128 andalso C3 =< 191, C4 >= 128 andalso C4 =< 191 -> 475 | tokenize_string(B, ?ADV_COL(S, 4), [C4, C3, C2, C1 | Acc]); 476 | _ -> 477 | error(invalid_utf8) 478 | end. 479 | 480 | tokenize_number(B, S) -> 481 | case tokenize_number(B, sign, S, []) of 482 | {{int, Int}, S1} -> 483 | {{const, list_to_integer(Int)}, S1}; 484 | {{float, Float}, S1} -> 485 | {{const, list_to_float(Float)}, S1} 486 | end. 487 | 488 | tokenize_number(B, sign, S = #decoder{offset = O}, []) -> 489 | case B of 490 | <<_:O/binary, $-, _/binary>> -> 491 | tokenize_number(B, int, ?INC_COL(S), [$-]); 492 | _ -> 493 | tokenize_number(B, int, S, []) 494 | end; 495 | tokenize_number(B, int, S = #decoder{offset = O}, Acc) -> 496 | case B of 497 | <<_:O/binary, $0, _/binary>> -> 498 | tokenize_number(B, frac, ?INC_COL(S), [$0 | Acc]); 499 | <<_:O/binary, C, _/binary>> when C >= $1 andalso C =< $9 -> 500 | tokenize_number(B, int1, ?INC_COL(S), [C | Acc]) 501 | end; 502 | tokenize_number(B, int1, S = #decoder{offset = O}, Acc) -> 503 | case B of 504 | <<_:O/binary, C, _/binary>> when C >= $0 andalso C =< $9 -> 505 | tokenize_number(B, int1, ?INC_COL(S), [C | Acc]); 506 | _ -> 507 | tokenize_number(B, frac, S, Acc) 508 | end; 509 | tokenize_number(B, frac, S = #decoder{offset = O}, Acc) -> 510 | case B of 511 | <<_:O/binary, $., C, _/binary>> when C >= $0, C =< $9 -> 512 | tokenize_number(B, frac1, ?ADV_COL(S, 2), [C, $. | Acc]); 513 | <<_:O/binary, E, _/binary>> when E =:= $e orelse E =:= $E -> 514 | tokenize_number(B, esign, ?INC_COL(S), [$e, $0, $. | Acc]); 515 | _ -> 516 | {{int, lists:reverse(Acc)}, S} 517 | end; 518 | tokenize_number(B, frac1, S = #decoder{offset = O}, Acc) -> 519 | case B of 520 | <<_:O/binary, C, _/binary>> when C >= $0 andalso C =< $9 -> 521 | tokenize_number(B, frac1, ?INC_COL(S), [C | Acc]); 522 | <<_:O/binary, E, _/binary>> when E =:= $e orelse E =:= $E -> 523 | tokenize_number(B, esign, ?INC_COL(S), [$e | Acc]); 524 | _ -> 525 | {{float, lists:reverse(Acc)}, S} 526 | end; 527 | tokenize_number(B, esign, S = #decoder{offset = O}, Acc) -> 528 | case B of 529 | <<_:O/binary, C, _/binary>> when C =:= $- orelse C =:= $+ -> 530 | tokenize_number(B, eint, ?INC_COL(S), [C | Acc]); 531 | _ -> 532 | tokenize_number(B, eint, S, Acc) 533 | end; 534 | tokenize_number(B, eint, S = #decoder{offset = O}, Acc) -> 535 | case B of 536 | <<_:O/binary, C, _/binary>> when C >= $0 andalso C =< $9 -> 537 | tokenize_number(B, eint1, ?INC_COL(S), [C | Acc]); 538 | _ -> 539 | exit({invalid_eint, B}) 540 | end; 541 | tokenize_number(B, eint1, S = #decoder{offset = O}, Acc) -> 542 | case B of 543 | <<_:O/binary, C, _/binary>> when C >= $0 andalso C =< $9 -> 544 | tokenize_number(B, eint1, ?INC_COL(S), [C | Acc]); 545 | _ -> 546 | {{float, lists:reverse(Acc)}, S} 547 | end. 548 | 549 | tokenize(B, S = #decoder{offset = O}) -> 550 | case B of 551 | <<_:O/binary, C, _/binary>> when ?IS_WHITESPACE(C) -> 552 | tokenize(B, ?INC_CHAR(S, C)); 553 | <<_:O/binary, "{", _/binary>> -> 554 | {start_object, ?INC_COL(S)}; 555 | <<_:O/binary, "}", _/binary>> -> 556 | {end_object, ?INC_COL(S)}; 557 | <<_:O/binary, "[", _/binary>> -> 558 | {start_array, ?INC_COL(S)}; 559 | <<_:O/binary, "]", _/binary>> -> 560 | {end_array, ?INC_COL(S)}; 561 | <<_:O/binary, ",", _/binary>> -> 562 | {comma, ?INC_COL(S)}; 563 | <<_:O/binary, ":", _/binary>> -> 564 | {colon, ?INC_COL(S)}; 565 | <<_:O/binary, "null", _/binary>> -> 566 | {{const, null}, ?ADV_COL(S, 4)}; 567 | <<_:O/binary, "true", _/binary>> -> 568 | {{const, true}, ?ADV_COL(S, 4)}; 569 | <<_:O/binary, "false", _/binary>> -> 570 | {{const, false}, ?ADV_COL(S, 5)}; 571 | <<_:O/binary, "\"", _/binary>> -> 572 | tokenize_string(B, ?INC_COL(S)); 573 | <<_:O/binary, C, _/binary>> when C >= $0 andalso C =< $9 orelse C =:= $- -> 574 | tokenize_number(B, S); 575 | <<_:O/binary>> -> 576 | trim = S#decoder.state, 577 | {eof, S} 578 | end. 579 | 580 | %% 581 | %% Tests 582 | %% 583 | -ifdef(TEST). 584 | 585 | -include_lib("eunit/include/eunit.hrl"). 586 | 587 | %% testing constructs borrowed from the Yaws JSON implementation. 588 | 589 | %% Create an object from a list of Key/Value pairs. 590 | 591 | obj_new() -> 592 | {struct, []}. 593 | 594 | is_obj({struct, Props}) -> 595 | F = fun({K, _}) when is_binary(K) -> true end, 596 | lists:all(F, Props). 597 | 598 | obj_from_list(Props) -> 599 | Obj = {struct, Props}, 600 | ?assert(is_obj(Obj)), 601 | Obj. 602 | 603 | %% Test for equivalence of Erlang terms. 604 | %% Due to arbitrary order of construction, equivalent objects might 605 | %% compare unequal as erlang terms, so we need to carefully recurse 606 | %% through aggregates (tuples and objects). 607 | 608 | equiv({struct, Props1}, {struct, Props2}) -> 609 | equiv_object(Props1, Props2); 610 | equiv(L1, L2) when is_list(L1), is_list(L2) -> 611 | equiv_list(L1, L2); 612 | equiv(N1, N2) when is_number(N1), is_number(N2) -> 613 | N1 == N2; 614 | equiv(B1, B2) when is_binary(B1), is_binary(B2) -> 615 | B1 == B2; 616 | equiv(A, A) when A =:= true orelse A =:= false orelse A =:= null -> 617 | true. 618 | 619 | %% Object representation and traversal order is unknown. 620 | %% Use the sledgehammer and sort property lists. 621 | 622 | equiv_object(Props1, Props2) -> 623 | L1 = lists:keysort(1, Props1), 624 | L2 = lists:keysort(1, Props2), 625 | Pairs = lists:zip(L1, L2), 626 | true = lists:all(fun({{K1, V1}, {K2, V2}}) -> equiv(K1, K2) and equiv(V1, V2) end, Pairs). 627 | 628 | %% Recursively compare tuple elements for equivalence. 629 | 630 | equiv_list([], []) -> 631 | true; 632 | equiv_list([V1 | L1], [V2 | L2]) -> 633 | equiv(V1, V2) andalso equiv_list(L1, L2). 634 | 635 | decode_test() -> 636 | [1199344435545.0, 1] = decode(<<"[1199344435545.0,1]">>), 637 | <<16#F0, 16#9D, 16#9C, 16#95>> = decode([34, "\\ud835", "\\udf15", 34]). 638 | 639 | e2j_vec_test() -> 640 | test_one(e2j_test_vec(utf8), 1). 641 | 642 | test_one([], _N) -> 643 | %% io:format("~p tests passed~n", [N-1]), 644 | ok; 645 | test_one([{E, J} | Rest], N) -> 646 | %% io:format("[~p] ~p ~p~n", [N, E, J]), 647 | true = equiv(E, decode(J)), 648 | true = equiv(E, decode(encode(E))), 649 | test_one(Rest, 1 + N). 650 | 651 | e2j_test_vec(utf8) -> 652 | [{1, "1"}, 653 | {3.1416, "3.14160"}, %% text representation may truncate, trail zeroes 654 | {-1, "-1"}, 655 | {-3.1416, "-3.14160"}, 656 | {12.0e10, "1.20000e+11"}, 657 | {1.234E+10, "1.23400e+10"}, 658 | {-1.234E-10, "-1.23400e-10"}, 659 | {10.0, "1.0e+01"}, 660 | {123.456, "1.23456E+2"}, 661 | {10.0, "1e1"}, 662 | {<<"foo">>, "\"foo\""}, 663 | {<<"foo", 5, "bar">>, "\"foo\\u0005bar\""}, 664 | {<<"">>, "\"\""}, 665 | {<<"\n\n\n">>, "\"\\n\\n\\n\""}, 666 | {<<"\" \b\f\r\n\t\"">>, "\"\\\" \\b\\f\\r\\n\\t\\\"\""}, 667 | {obj_new(), "{}"}, 668 | {obj_from_list([{<<"foo">>, <<"bar">>}]), "{\"foo\":\"bar\"}"}, 669 | {obj_from_list([{<<"foo">>, <<"bar">>}, {<<"baz">>, 123}]), 670 | "{\"foo\":\"bar\",\"baz\":123}"}, 671 | {[], "[]"}, 672 | {[[]], "[[]]"}, 673 | {[1, <<"foo">>], "[1,\"foo\"]"}, 674 | %% json array in a json object 675 | {obj_from_list([{<<"foo">>, [123]}]), "{\"foo\":[123]}"}, 676 | %% json object in a json object 677 | {obj_from_list([{<<"foo">>, obj_from_list([{<<"bar">>, true}])}]), 678 | "{\"foo\":{\"bar\":true}}"}, 679 | %% fold evaluation order 680 | {obj_from_list([{<<"foo">>, []}, 681 | {<<"bar">>, obj_from_list([{<<"baz">>, true}])}, 682 | {<<"alice">>, <<"bob">>}]), 683 | "{\"foo\":[],\"bar\":{\"baz\":true},\"alice\":\"bob\"}"}, 684 | %% json object in a json array 685 | {[-123, <<"foo">>, obj_from_list([{<<"bar">>, []}]), null], 686 | "[-123,\"foo\",{\"bar\":[]},null]"}]. 687 | 688 | %% test utf8 encoding 689 | encoder_utf8_test() -> 690 | %% safe conversion case (default) 691 | [34, "\\u0001", "\\u0442", "\\u0435", "\\u0441", "\\u0442", 34] = 692 | encode(<<1, "\321\202\320\265\321\201\321\202">>), 693 | 694 | %% raw utf8 output (optional) 695 | Enc = dmochijson2:encoder([{utf8, true}]), 696 | [34, "\\u0001", [209, 130], [208, 181], [209, 129], [209, 130], 34] = 697 | Enc(<<1, "\321\202\320\265\321\201\321\202">>). 698 | 699 | input_validation_test() -> 700 | Good = 701 | [{16#00A3, <>}, %% pound 702 | {16#20AC, <>}, %% euro 703 | {16#10196, <>}], %% denarius 704 | lists:foreach(fun({CodePoint, UTF8}) -> 705 | Expect = list_to_binary(xmerl_ucs:to_utf8(CodePoint)), 706 | Expect = decode(UTF8) 707 | end, 708 | Good), 709 | 710 | Bad = [%% 2nd, 3rd, or 4th byte of a multi-byte sequence w/o leading byte 711 | <>, 712 | %% missing continuations, last byte in each should be 80-BF 713 | <>, 714 | <>, 715 | <>, 716 | %% we don't support code points > 10FFFF per RFC 3629 717 | <>, 718 | %% escape characters trigger a different code path 719 | <>], 720 | lists:foreach(fun(X) -> 721 | ok = 722 | try 723 | decode(X) 724 | catch 725 | _:invalid_utf8 -> 726 | ok 727 | end, 728 | try encode(X) of 729 | Result -> 730 | exit({unexpected_result, Result}) 731 | catch 732 | _:Error -> 733 | %% could be {ucs,{bad_utf8_character_code}} or 734 | %% {json_encode,{bad_char,_}} 735 | ?assert(is_tuple(Error)) 736 | end 737 | end, 738 | Bad). 739 | 740 | inline_json_test() -> 741 | ?assertEqual(<<"\"iodata iodata\"">>, 742 | iolist_to_binary(encode({json, [<<"\"iodata">>, " iodata\""]}))), 743 | ?assertEqual({struct, [{<<"key">>, <<"iodata iodata">>}]}, 744 | decode(encode({struct, [{key, {json, [<<"\"iodata">>, " iodata\""]}}]}))), 745 | ok. 746 | 747 | big_unicode_test() -> 748 | UTF8Seq = list_to_binary(xmerl_ucs:to_utf8(16#0001d120)), 749 | ?assertEqual(<<"\"\\ud834\\udd20\"">>, iolist_to_binary(encode(UTF8Seq))), 750 | ?assertEqual(UTF8Seq, decode(iolist_to_binary(encode(UTF8Seq)))), 751 | ok. 752 | 753 | custom_decoder_test() -> 754 | ?assertEqual({struct, [{<<"key">>, <<"value">>}]}, (decoder([]))("{\"key\": \"value\"}")), 755 | F = fun({struct, [{<<"key">>, <<"value">>}]}) -> win end, 756 | ?assertEqual(win, (decoder([{object_hook, F}]))("{\"key\": \"value\"}")), 757 | ok. 758 | 759 | atom_test() -> 760 | %% JSON native atoms 761 | lists:foreach(fun(A) -> 762 | ?assertEqual(A, decode(atom_to_list(A))), 763 | ?assertEqual(iolist_to_binary(atom_to_list(A)), iolist_to_binary(encode(A))) 764 | end, 765 | [true, false, null]), 766 | %% Atom to string 767 | ?assertEqual(<<"\"foo\"">>, iolist_to_binary(encode(foo))), 768 | ?assertEqual(<<"\"\\ud834\\udd20\"">>, 769 | iolist_to_binary(encode(list_to_atom(xmerl_ucs:to_utf8(16#0001d120))))), 770 | ok. 771 | 772 | key_encode_test() -> 773 | %% Some forms are accepted as keys that would not be strings in other 774 | %% cases 775 | ?assertEqual(<<"{\"foo\":1}">>, iolist_to_binary(encode({struct, [{foo, 1}]}))), 776 | ?assertEqual(<<"{\"foo\":1}">>, iolist_to_binary(encode({struct, [{<<"foo">>, 1}]}))), 777 | ?assertEqual(<<"{\"foo\":1}">>, iolist_to_binary(encode({struct, [{"foo", 1}]}))), 778 | ?assertEqual(<<"{\"foo\":1}">>, iolist_to_binary(encode([{foo, 1}]))), 779 | ?assertEqual(<<"{\"foo\":1}">>, iolist_to_binary(encode([{<<"foo">>, 1}]))), 780 | ?assertEqual(<<"{\"foo\":1}">>, iolist_to_binary(encode([{"foo", 1}]))), 781 | ?assertEqual(<<"{\"\\ud834\\udd20\":1}">>, 782 | iolist_to_binary(encode({struct, [{[16#0001d120], 1}]}))), 783 | ?assertEqual(<<"{\"1\":1}">>, iolist_to_binary(encode({struct, [{1, 1}]}))), 784 | ok. 785 | 786 | unsafe_chars_test() -> 787 | Chars = "\"\\\b\f\n\r\t", 788 | lists:foreach(fun(C) -> 789 | ?assertEqual(false, json_string_is_safe([C])), 790 | ?assertEqual(false, json_bin_is_safe(<>)), 791 | ?assertEqual(<>, decode(encode(<>))) 792 | end, 793 | Chars), 794 | ?assertEqual(false, json_string_is_safe([16#0001d120])), 795 | ?assertEqual(false, json_bin_is_safe(list_to_binary(xmerl_ucs:to_utf8(16#0001d120)))), 796 | ?assertEqual([16#0001d120], 797 | xmerl_ucs:from_utf8(binary_to_list(decode(encode(list_to_atom(xmerl_ucs:to_utf8(16#0001d120))))))), 798 | ?assertEqual(false, json_string_is_safe([16#110000])), 799 | ?assertEqual(false, json_bin_is_safe(list_to_binary(xmerl_ucs:to_utf8([16#110000])))), 800 | %% solidus can be escaped but isn't unsafe by default 801 | ?assertEqual(<<"/">>, decode(<<"\"\\/\"">>)), 802 | ok. 803 | 804 | int_test() -> 805 | ?assertEqual(0, decode("0")), 806 | ?assertEqual(1, decode("1")), 807 | ?assertEqual(11, decode("11")), 808 | ok. 809 | 810 | large_int_test() -> 811 | ?assertEqual(<<"-2147483649214748364921474836492147483649">>, 812 | iolist_to_binary(encode(-2147483649214748364921474836492147483649))), 813 | ?assertEqual(<<"2147483649214748364921474836492147483649">>, 814 | iolist_to_binary(encode(2147483649214748364921474836492147483649))), 815 | ok. 816 | 817 | float_test() -> 818 | ?assertEqual(<<"-2147483649.0">>, iolist_to_binary(encode(-2147483649.0))), 819 | ?assertEqual(<<"2147483648.0">>, iolist_to_binary(encode(2147483648.0))), 820 | ok. 821 | 822 | handler_test() -> 823 | ?assertExit({json_encode, {bad_term, {x, y}}}, encode({x, y})), 824 | F = fun({x, y}) -> [] end, 825 | ?assertEqual(<<"[]">>, iolist_to_binary((encoder([{handler, F}]))({x, y}))), 826 | ok. 827 | 828 | encode_empty_test_() -> 829 | [{A, ?_assertEqual(<<"{}">>, iolist_to_binary(encode(B)))} 830 | || {A, B} <- [{"eep18 {}", {}}, {"eep18 {[]}", {[]}}, {"{struct, []}", {struct, []}}]]. 831 | 832 | encode_test_() -> 833 | P = [{<<"k">>, <<"v">>}], 834 | JSON = iolist_to_binary(encode({struct, P})), 835 | [{atom_to_list(F), 836 | ?_assertEqual(JSON, iolist_to_binary(encode(decode(JSON, [{format, F}]))))} 837 | || F <- [struct, eep18, proplist]]. 838 | 839 | format_test_() -> 840 | P = [{<<"k">>, <<"v">>}], 841 | JSON = iolist_to_binary(encode({struct, P})), 842 | [{atom_to_list(F), ?_assertEqual(A, decode(JSON, [{format, F}]))} 843 | || {F, A} <- [{struct, {struct, P}}, {eep18, {P}}, {proplist, P}]]. 844 | 845 | -endif. 846 | -------------------------------------------------------------------------------- /src/dmochinum.erl: -------------------------------------------------------------------------------- 1 | %% @copyright 2007 Mochi Media, Inc. 2 | %% @author Bob Ippolito 3 | 4 | %% @doc Useful numeric algorithms for floats that cover some deficiencies 5 | %% in the math module. More interesting is digits/1, which implements 6 | %% the algorithm from: 7 | %% http://www.cs.indiana.edu/~burger/fp/index.html 8 | %% See also "Printing Floating-Point Numbers Quickly and Accurately" 9 | %% in Proceedings of the SIGPLAN '96 Conference on Programming Language 10 | %% Design and Implementation. 11 | 12 | -module(dmochinum). 13 | 14 | -export([digits/1, frexp/1, int_pow/2, int_ceil/1]). 15 | 16 | %% IEEE 754 Float exponent bias 17 | -define(FLOAT_BIAS, 1022). 18 | -define(MIN_EXP, -1074). 19 | -define(BIG_POW, 4503599627370496). 20 | 21 | %% External API 22 | 23 | %% @spec digits(number()) -> string() 24 | %% @doc Returns a string that accurately represents the given integer or float 25 | %% using a conservative amount of digits. Great for generating 26 | %% human-readable output, or compact ASCII serializations for floats. 27 | digits(N) when is_integer(N) -> 28 | integer_to_list(N); 29 | digits(N) when N == +0.0; N == -0.0 -> 30 | "0.0"; 31 | digits(Float) -> 32 | {Frac1, Exp1} = frexp_int(Float), 33 | [Place0 | Digits0] = digits1(Float, Exp1, Frac1), 34 | {Place, Digits} = transform_digits(Place0, Digits0), 35 | R = insert_decimal(Place, Digits), 36 | case Float < 0 of 37 | true -> 38 | [$- | R]; 39 | _ -> 40 | R 41 | end. 42 | 43 | %% @spec frexp(F::float()) -> {Frac::float(), Exp::float()} 44 | %% @doc Return the fractional and exponent part of an IEEE 754 double, 45 | %% equivalent to the libc function of the same name. 46 | %% F = Frac * pow(2, Exp). 47 | frexp(F) -> 48 | frexp1(unpack(F)). 49 | 50 | %% @spec int_pow(X::integer(), N::integer()) -> Y::integer() 51 | %% @doc Moderately efficient way to exponentiate integers. 52 | %% int_pow(10, 2) = 100. 53 | int_pow(_X, 0) -> 54 | 1; 55 | int_pow(X, N) when N > 0 -> 56 | int_pow(X, N, 1). 57 | 58 | %% @spec int_ceil(F::float()) -> integer() 59 | %% @doc Return the ceiling of F as an integer. The ceiling is defined as 60 | %% F when F == trunc(F); 61 | %% trunc(F) when F < 0; 62 | %% trunc(F) + 1 when F > 0. 63 | int_ceil(X) -> 64 | T = trunc(X), 65 | case X - T of 66 | Pos when Pos > 0 -> 67 | T + 1; 68 | _ -> 69 | T 70 | end. 71 | 72 | %% Internal API 73 | 74 | int_pow(X, N, R) when N < 2 -> 75 | R * X; 76 | int_pow(X, N, R) -> 77 | int_pow(X * X, 78 | N bsr 1, 79 | case N band 1 of 80 | 1 -> 81 | R * X; 82 | 0 -> 83 | R 84 | end). 85 | 86 | insert_decimal(0, S) -> 87 | "0." ++ S; 88 | insert_decimal(Place, S) when Place > 0 -> 89 | L = length(S), 90 | case Place - L of 91 | 0 -> 92 | S ++ ".0"; 93 | N when N < 0 -> 94 | {S0, S1} = lists:split(L + N, S), 95 | S0 ++ "." ++ S1; 96 | N when N < 6 -> 97 | %% More places than digits 98 | S ++ lists:duplicate(N, $0) ++ ".0"; 99 | _ -> 100 | insert_decimal_exp(Place, S) 101 | end; 102 | insert_decimal(Place, S) when Place > -6 -> 103 | "0." ++ lists:duplicate(abs(Place), $0) ++ S; 104 | insert_decimal(Place, S) -> 105 | insert_decimal_exp(Place, S). 106 | 107 | insert_decimal_exp(Place, S) -> 108 | [C | S0] = S, 109 | S1 = case S0 of 110 | [] -> 111 | "0"; 112 | _ -> 113 | S0 114 | end, 115 | Exp = case Place < 0 of 116 | true -> 117 | "e-"; 118 | false -> 119 | "e+" 120 | end, 121 | [C] ++ "." ++ S1 ++ Exp ++ integer_to_list(abs(Place - 1)). 122 | 123 | digits1(Float, Exp, Frac) -> 124 | Round = Frac band 1 =:= 0, 125 | case Exp >= 0 of 126 | true -> 127 | BExp = 1 bsl Exp, 128 | case Frac =/= ?BIG_POW of 129 | true -> 130 | scale(Frac * BExp * 2, 2, BExp, BExp, Round, Round, Float); 131 | false -> 132 | scale(Frac * BExp * 4, 4, BExp * 2, BExp, Round, Round, Float) 133 | end; 134 | false -> 135 | case Exp =:= ?MIN_EXP orelse Frac =/= ?BIG_POW of 136 | true -> 137 | scale(Frac * 2, 1 bsl (1 - Exp), 1, 1, Round, Round, Float); 138 | false -> 139 | scale(Frac * 4, 1 bsl (2 - Exp), 2, 1, Round, Round, Float) 140 | end 141 | end. 142 | 143 | scale(R, S, MPlus, MMinus, LowOk, HighOk, Float) -> 144 | Est = int_ceil(math:log10(abs(Float)) - 1.0e-10), 145 | %% Note that the scheme implementation uses a 326 element look-up table 146 | %% for int_pow(10, N) where we do not. 147 | case Est >= 0 of 148 | true -> 149 | fixup(R, S * int_pow(10, Est), MPlus, MMinus, Est, LowOk, HighOk); 150 | false -> 151 | Scale = int_pow(10, -Est), 152 | fixup(R * Scale, S, MPlus * Scale, MMinus * Scale, Est, LowOk, HighOk) 153 | end. 154 | 155 | fixup(R, S, MPlus, MMinus, K, LowOk, HighOk) -> 156 | TooLow = 157 | case HighOk of 158 | true -> 159 | R + MPlus >= S; 160 | false -> 161 | R + MPlus > S 162 | end, 163 | case TooLow of 164 | true -> 165 | [K + 1 | generate(R, S, MPlus, MMinus, LowOk, HighOk)]; 166 | false -> 167 | [K | generate(R * 10, S, MPlus * 10, MMinus * 10, LowOk, HighOk)] 168 | end. 169 | 170 | generate(R0, S, MPlus, MMinus, LowOk, HighOk) -> 171 | D = R0 div S, 172 | R = R0 rem S, 173 | TC1 = case LowOk of 174 | true -> 175 | R =< MMinus; 176 | false -> 177 | R < MMinus 178 | end, 179 | TC2 = case HighOk of 180 | true -> 181 | R + MPlus >= S; 182 | false -> 183 | R + MPlus > S 184 | end, 185 | case TC1 of 186 | false -> 187 | case TC2 of 188 | false -> 189 | [D | generate(R * 10, S, MPlus * 10, MMinus * 10, LowOk, HighOk)]; 190 | true -> 191 | [D + 1] 192 | end; 193 | true -> 194 | case TC2 of 195 | false -> 196 | [D]; 197 | true -> 198 | case R * 2 < S of 199 | true -> 200 | [D]; 201 | false -> 202 | [D + 1] 203 | end 204 | end 205 | end. 206 | 207 | unpack(Float) -> 208 | <> = <>, 209 | {Sign, Exp, Frac}. 210 | 211 | frexp1({_Sign, 0, 0}) -> 212 | {0.0, 0}; 213 | frexp1({Sign, 0, Frac}) -> 214 | Exp = log2floor(Frac), 215 | <> = <>, 216 | {Frac1, -?FLOAT_BIAS - 52 + Exp}; 217 | frexp1({Sign, Exp, Frac}) -> 218 | <> = <>, 219 | {Frac1, Exp - ?FLOAT_BIAS}. 220 | 221 | log2floor(Int) -> 222 | log2floor(Int, 0). 223 | 224 | log2floor(0, N) -> 225 | N; 226 | log2floor(Int, N) -> 227 | log2floor(Int bsr 1, 1 + N). 228 | 229 | transform_digits(Place, [0 | Rest]) -> 230 | transform_digits(Place, Rest); 231 | transform_digits(Place, Digits) -> 232 | {Place, [$0 + D || D <- Digits]}. 233 | 234 | frexp_int(F) -> 235 | case unpack(F) of 236 | {_Sign, 0, Frac} -> 237 | {Frac, ?MIN_EXP}; 238 | {_Sign, Exp, Frac} -> 239 | {Frac + (1 bsl 52), Exp - 53 - ?FLOAT_BIAS} 240 | end. 241 | 242 | %% 243 | %% Tests 244 | %% 245 | -ifdef(TEST). 246 | 247 | -include_lib("eunit/include/eunit.hrl"). 248 | 249 | int_ceil_test() -> 250 | ?assertEqual(1, int_ceil(0.0001)), 251 | ?assertEqual(0, int_ceil(0.0)), 252 | ?assertEqual(1, int_ceil(0.99)), 253 | ?assertEqual(1, int_ceil(1.0)), 254 | ?assertEqual(-1, int_ceil(-1.5)), 255 | ?assertEqual(-2, int_ceil(-2.0)), 256 | ok. 257 | 258 | int_pow_test() -> 259 | ?assertEqual(1, int_pow(1, 1)), 260 | ?assertEqual(1, int_pow(1, 0)), 261 | ?assertEqual(1, int_pow(10, 0)), 262 | ?assertEqual(10, int_pow(10, 1)), 263 | ?assertEqual(100, int_pow(10, 2)), 264 | ?assertEqual(1000, int_pow(10, 3)), 265 | ok. 266 | 267 | digits_test() -> 268 | ?assertEqual("0", digits(0)), 269 | ?assertEqual("0.0", digits(0.0)), 270 | ?assertEqual("1.0", digits(1.0)), 271 | ?assertEqual("-1.0", digits(-1.0)), 272 | ?assertEqual("0.1", digits(0.1)), 273 | ?assertEqual("0.01", digits(0.01)), 274 | ?assertEqual("0.001", digits(0.001)), 275 | ?assertEqual("1.0e+6", digits(1000000.0)), 276 | ?assertEqual("0.5", digits(0.5)), 277 | ?assertEqual("4503599627370496.0", digits(4503599627370496.0)), 278 | %% small denormalized number 279 | %% 4.94065645841246544177e-324 =:= 5.0e-324 280 | <> = <<0, 0, 0, 0, 0, 0, 0, 1>>, 281 | ?assertEqual("5.0e-324", digits(SmallDenorm)), 282 | ?assertEqual(SmallDenorm, list_to_float(digits(SmallDenorm))), 283 | %% large denormalized number 284 | %% 2.22507385850720088902e-308 285 | <> = <<0, 15, 255, 255, 255, 255, 255, 255>>, 286 | ?assertEqual("2.225073858507201e-308", digits(BigDenorm)), 287 | ?assertEqual(BigDenorm, list_to_float(digits(BigDenorm))), 288 | %% small normalized number 289 | %% 2.22507385850720138309e-308 290 | <> = <<0, 16, 0, 0, 0, 0, 0, 0>>, 291 | ?assertEqual("2.2250738585072014e-308", digits(SmallNorm)), 292 | ?assertEqual(SmallNorm, list_to_float(digits(SmallNorm))), 293 | %% large normalized number 294 | %% 1.79769313486231570815e+308 295 | <> = <<127, 239, 255, 255, 255, 255, 255, 255>>, 296 | ?assertEqual("1.7976931348623157e+308", digits(LargeNorm)), 297 | ?assertEqual(LargeNorm, list_to_float(digits(LargeNorm))), 298 | %% issue #10 - mochinum:frexp(math:pow(2, -1074)). 299 | ?assertEqual("5.0e-324", digits(math:pow(2, -1074))), 300 | ok. 301 | 302 | frexp_test() -> 303 | %% zero 304 | ?assertEqual({0.0, 0}, frexp(0.0)), 305 | %% one 306 | ?assertEqual({0.5, 1}, frexp(1.0)), 307 | %% negative one 308 | ?assertEqual({-0.5, 1}, frexp(-1.0)), 309 | %% small denormalized number 310 | %% 4.94065645841246544177e-324 311 | <> = <<0, 0, 0, 0, 0, 0, 0, 1>>, 312 | ?assertEqual({0.5, -1073}, frexp(SmallDenorm)), 313 | %% large denormalized number 314 | %% 2.22507385850720088902e-308 315 | <> = <<0, 15, 255, 255, 255, 255, 255, 255>>, 316 | ?assertEqual({0.99999999999999978, -1022}, frexp(BigDenorm)), 317 | %% small normalized number 318 | %% 2.22507385850720138309e-308 319 | <> = <<0, 16, 0, 0, 0, 0, 0, 0>>, 320 | ?assertEqual({0.5, -1021}, frexp(SmallNorm)), 321 | %% large normalized number 322 | %% 1.79769313486231570815e+308 323 | <> = <<127, 239, 255, 255, 255, 255, 255, 255>>, 324 | ?assertEqual({0.99999999999999989, 1024}, frexp(LargeNorm)), 325 | %% issue #10 - mochinum:frexp(math:pow(2, -1074)). 326 | ?assertEqual({0.5, -1073}, frexp(math:pow(2, -1074))), 327 | ok. 328 | 329 | -endif. 330 | -------------------------------------------------------------------------------- /src/dynamodb.erl: -------------------------------------------------------------------------------- 1 | -module(dynamodb). 2 | 3 | -type aws_datetime() :: string(). 4 | -type endpoint() :: string(). 5 | -type header() :: {string() | atom(), string()}. 6 | -type headers() :: [header()]. 7 | 8 | -export_type([aws_datetime/0]). 9 | 10 | -export([call/5, call/6]). 11 | 12 | %% @todo Remove when lhttpc's specs are fixed 13 | -dialyzer({nowarn_function, [call/6, submit/4]}). 14 | 15 | -spec endpoint(dinerl:zone()) -> endpoint(). 16 | endpoint("us-east-1" ++ _R) -> 17 | "dynamodb.us-east-1.amazonaws.com"; 18 | endpoint("us-west-1" ++ _R) -> 19 | "dynamodb.us-west-1.amazonaws.com"; 20 | endpoint("us-west-2" ++ _R) -> 21 | "dynamodb.us-west-2.amazonaws.com"; 22 | endpoint("ap-northeast-1" ++ _R) -> 23 | "dynamodb.ap-northeast-1.amazonaws.com"; 24 | endpoint("ap-southeast-1" ++ _R) -> 25 | "dynamodb.ap-southeast-1.amazonaws.com"; 26 | endpoint("eu-west-1" ++ _R) -> 27 | "dynamodb.eu-west-1.amazonaws.com". 28 | 29 | -spec region(dinerl:zone()) -> string(). 30 | region("us-east-1" ++ _R) -> 31 | "us-east-1"; 32 | region("us-west-1" ++ _R) -> 33 | "us-west-1"; 34 | region("us-west-2" ++ _R) -> 35 | "us-west-2"; 36 | region("ap-northeast-1" ++ _R) -> 37 | "ap-northeast-1"; 38 | region("ap-southeast-1" ++ _R) -> 39 | "ap-southeast-1"; 40 | region("eu-west-1" ++ _R) -> 41 | "eu-west-1". 42 | 43 | -spec call(awsv4:credentials(), 44 | dinerl:zone(), 45 | string(), 46 | aws_datetime(), 47 | iodata(), 48 | undefined | pos_integer()) -> 49 | dinerl:result(). 50 | call(Credentials, Zone, Target, ISODate, Body, undefined) -> 51 | call(Credentials, Zone, Target, ISODate, Body, 1000); 52 | call(Credentials, Zone, Target, ISODate, Body, Timeout) -> 53 | Host = endpoint(Zone), 54 | Headers = 55 | awsv4:headers(Credentials, 56 | #{service => "dynamodb", 57 | target_api => Target, 58 | method => "POST", 59 | aws_date => ISODate, 60 | host => Host, 61 | region => region(Zone)}, 62 | Body), 63 | submit(Host, [{"content-type", "application/x-amz-json-1.0"} | Headers], Body, Timeout). 64 | 65 | call(Credentials, Zone, Target, RFCDate, Body) -> 66 | call(Credentials, Zone, Target, RFCDate, Body, 1000). 67 | 68 | -spec submit(endpoint(), headers(), iodata(), integer()) -> dinerl:result(). 69 | submit(Host, Headers, Body, Timeout) when is_list(Host) -> 70 | MaxConnections = dinerl_util:get_env(max_connections), 71 | Opts = [{max_connections, MaxConnections}], 72 | Endpoint = "http://" ++ Host ++ "/", 73 | dinerl_util:increment([dinerl, dynamodb, call, {endpoint, list_to_atom(Host)}]), 74 | F = fun() -> lhttpc:request(Endpoint, "POST", Headers, Body, Timeout, Opts) end, 75 | case dinerl_util:time_call([dinerl, dynamodb, call, time, list_to_atom(Host)], F) of 76 | {ok, {{200, _}, _Headers, Response}} -> 77 | dinerl_util:increment([dinerl, 78 | dynamodb, 79 | call, 80 | result, 81 | {endpoint, list_to_atom(Host)}, 82 | {result, ok}]), 83 | {ok, Response}; 84 | {ok, {{400, Code}, _Headers, ErrorString}} -> 85 | dinerl_util:increment([dinerl, 86 | dynamodb, 87 | call, 88 | result, 89 | {endpoint, list_to_atom(Host)}, 90 | {result, '400'}]), 91 | {error, Code, ErrorString}; 92 | {ok, {{413, Code}, _Headers, ErrorString}} -> 93 | dinerl_util:increment([dinerl, 94 | dynamodb, 95 | call, 96 | result, 97 | {endpoint, list_to_atom(Host)}, 98 | {result, '413'}]), 99 | {error, Code, ErrorString}; 100 | {ok, {{500, Code}, _Headers, ErrorString}} -> 101 | dinerl_util:increment([dinerl, 102 | dynamodb, 103 | call, 104 | result, 105 | {endpoint, list_to_atom(Host)}, 106 | {result, '500'}]), 107 | {error, Code, ErrorString}; 108 | {error, Reason} -> 109 | dinerl_util:increment([dinerl, 110 | dynamodb, 111 | call, 112 | result, 113 | {endpoint, list_to_atom(Host)}, 114 | {result, error}]), 115 | {error, unknown, Reason}; 116 | Other -> 117 | dinerl_util:increment([dinerl, 118 | dynamodb, 119 | call, 120 | result, 121 | {endpoint, list_to_atom(Host)}, 122 | {result, unknown}]), 123 | {error, response, Other} 124 | end. 125 | --------------------------------------------------------------------------------