├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── ebin └── .gitignore ├── rebar ├── rebar.config ├── rebar.config.script ├── src ├── mnesia_eleveldb.app.src ├── mnesia_eleveldb.erl ├── mnesia_eleveldb_app.erl ├── mnesia_eleveldb_params.erl ├── mnesia_eleveldb_sext.erl ├── mnesia_eleveldb_sup.erl ├── mnesia_eleveldb_tuning.erl └── mnesia_eleveldb_tuning.hrl └── test ├── Emakefile ├── README ├── basho_bench_driver_mnesia_eleveldb.erl ├── mnesia_eleveldb_basho_bench.hrl ├── mnesia_eleveldb_bench_disc_only.config ├── mnesia_eleveldb_bench_leveldb_copies.config ├── mnesia_eleveldb_chg_tbl_copy.erl ├── mnesia_eleveldb_conv_bigtab.erl ├── mnesia_eleveldb_fallback.erl ├── mnesia_eleveldb_indexes.erl ├── mnesia_eleveldb_proper_semantics_test.erl ├── mnesia_eleveldb_size_info.erl ├── mnesia_eleveldb_tlib.erl └── mnesia_eleveldb_xform.erl /.gitignore: -------------------------------------------------------------------------------- 1 | /doc 2 | /deps -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | suite=$(if $(SUITE), suite=$(SUITE), ) 2 | 3 | .PHONY: all deps check test clean 4 | 5 | all: deps 6 | ./rebar compile 7 | 8 | deps: 9 | ./rebar get-deps 10 | 11 | docs: 12 | ./rebar doc 13 | 14 | check: 15 | ./rebar check-plt 16 | ./rebar dialyze 17 | 18 | test: 19 | ./rebar eunit $(suite) skip_deps=true 20 | 21 | 22 | conf_clean: 23 | @: 24 | 25 | clean: 26 | ./rebar clean 27 | $(RM) doc/* 28 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # mnesia_eleveldb 2 | An eleveldb backend for Mnesia 3 | 4 | This permits Erlang/OTP applications to use LevelDB as a backend for 5 | mnesia tables. 6 | 7 | ## Prerequisites 8 | - eleveldb (https://github.com/basho/eleveldb) 9 | - Erlang/OTP 21.0 or newer (https://github.com/erlang/otp) 10 | 11 | ## Getting started 12 | 13 | Call `mnesia_eleveldb:register()` immediately after 14 | starting mnesia. 15 | 16 | Put `{leveldb_copies, [node()]}` into the table definitions of 17 | tables you want to be in LevelDB. 18 | 19 | ## Special features 20 | 21 | LevelDB tables support efficient selects on *prefix keys*. 22 | 23 | The backend uses the `mnesia_eleveldb_sext` module (see 24 | https://github.com/uwiger/sext) for mapping between Erlang terms and the 25 | binary data stored in the tables. This provides two useful properties: 26 | 27 | - The records are stored in the Erlang term order of their keys. 28 | - A prefix of a composite key is ordered just before any key for which 29 | it is a prefix. For example, `{x, '_'}` is a prefix for keys `{x, a}`, 30 | `{x, b}` and so on. 31 | 32 | This means that a prefix key identifies the start of the sequence of 33 | entries whose keys match the prefix. The backend uses this to optimize 34 | selects on prefix keys. 35 | 36 | ## Caveats 37 | 38 | Avoid placing `bag` tables in LevelDB. Although they work, each write 39 | requires additional reads, causing substantial runtime overheads. There 40 | are better ways to represent and process bag data (see above about 41 | *prefix keys*). 42 | 43 | The `mnesia:table_info(T, size)` call always returns zero for LevelDB 44 | tables. LevelDB itself does not track the number of elements in a table, and 45 | although it is possible to make the mnesia_eleveldb backend maintain a size 46 | counter, it incurs a high runtime overhead for writes and deletes since it 47 | forces them to first do a read to check the existence of the key. If you 48 | depend on having an up to date size count at all times, you need to maintain 49 | it yourself. If you only need the size occasionally, you may traverse the 50 | table to count the elements. 51 | -------------------------------------------------------------------------------- /ebin/.gitignore: -------------------------------------------------------------------------------- 1 | *.app 2 | *.beam 3 | *.d 4 | -------------------------------------------------------------------------------- /rebar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klarna/mnesia_eleveldb/af6d0556a78aec2918b3471f0c85121402a1f5b1/rebar -------------------------------------------------------------------------------- /rebar.config: -------------------------------------------------------------------------------- 1 | %% -*- erlang -*- 2 | {deps, 3 | [ {proper,".*", 4 | { git, 5 | "https://github.com/manopapad/proper.git", 6 | {branch, master} } } 7 | , {eleveldb,".*", 8 | { git, 9 | "https://github.com/basho/eleveldb.git", 10 | {branch, develop} } } 11 | ]}. 12 | -------------------------------------------------------------------------------- /rebar.config.script: -------------------------------------------------------------------------------- 1 | %% -*- erlang-mode -*- 2 | case os:getenv("DEBUG") of 3 | "true" -> 4 | Opts = proplists:get_value(erl_opts, CONFIG, []), 5 | lists:keystore(erl_opts, 1, CONFIG, 6 | [{d,'DEBUG'} | Opts -- [{d,'DEBUG'}]]); 7 | _ -> 8 | CONFIG 9 | end. 10 | -------------------------------------------------------------------------------- /src/mnesia_eleveldb.app.src: -------------------------------------------------------------------------------- 1 | {application, mnesia_eleveldb, 2 | [ 3 | {description, "eleveldb backend plugin for Mnesia"}, 4 | {vsn, "1.0"}, 5 | {modules, [mnesia_eleveldb, mnesia_eleveldb_app, 6 | mnesia_eleveldb_params, mnesia_eleveldb_sext, 7 | mnesia_eleveldb_sup, mnesia_eleveldb_tuning]}, 8 | {registered, []}, 9 | {mod, {mnesia_eleveldb_app, []}}, 10 | {env, []}, 11 | {applications, [kernel, stdlib]} 12 | ]}. 13 | -------------------------------------------------------------------------------- /src/mnesia_eleveldb.erl: -------------------------------------------------------------------------------- 1 | %%---------------------------------------------------------------- 2 | %% Copyright (c) 2013-2016 Klarna AB 3 | %% 4 | %% This file is provided to you under the Apache License, 5 | %% Version 2.0 (the "License"); you may not use this file 6 | %% except in compliance with the License. You may obtain 7 | %% a copy of the License at 8 | %% 9 | %% http://www.apache.org/licenses/LICENSE-2.0 10 | %% 11 | %% Unless required by applicable law or agreed to in writing, 12 | %% software distributed under the License is distributed on an 13 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | %% KIND, either express or implied. See the License for the 15 | %% specific language governing permissions and limitations 16 | %% under the License. 17 | %%---------------------------------------------------------------- 18 | 19 | %% @doc eleveldb storage backend for Mnesia. 20 | 21 | %% Initialization: register() or register(Alias) 22 | %% Usage: mnesia:create_table(Tab, [{leveldb_copies, Nodes}, ...]). 23 | 24 | -module(mnesia_eleveldb). 25 | 26 | 27 | %% ---------------------------------------------------------------------------- 28 | %% BEHAVIOURS 29 | %% ---------------------------------------------------------------------------- 30 | 31 | -behaviour(mnesia_backend_type). 32 | -behaviour(gen_server). 33 | 34 | 35 | %% ---------------------------------------------------------------------------- 36 | %% EXPORTS 37 | %% ---------------------------------------------------------------------------- 38 | 39 | %% 40 | %% CONVENIENCE API 41 | %% 42 | 43 | -export([register/0, 44 | register/1, 45 | default_alias/0]). 46 | 47 | %% 48 | %% DEBUG API 49 | %% 50 | 51 | -export([show_table/1, 52 | show_table/2, 53 | show_table/3, 54 | fold/6]). 55 | 56 | %% 57 | %% BACKEND CALLBACKS 58 | %% 59 | 60 | %% backend management 61 | -export([init_backend/0, 62 | add_aliases/1, 63 | remove_aliases/1]). 64 | 65 | %% schema level callbacks 66 | -export([semantics/2, 67 | check_definition/4, 68 | create_table/3, 69 | load_table/4, 70 | close_table/2, 71 | sync_close_table/2, 72 | delete_table/2, 73 | info/3]). 74 | 75 | %% table synch calls 76 | -export([sender_init/4, 77 | sender_handle_info/5, 78 | receiver_first_message/4, 79 | receive_data/5, 80 | receive_done/4]). 81 | 82 | %% low-level accessor callbacks. 83 | -export([delete/3, 84 | first/2, 85 | fixtable/3, 86 | insert/3, 87 | last/2, 88 | lookup/3, 89 | match_delete/3, 90 | next/3, 91 | prev/3, 92 | repair_continuation/2, 93 | select/1, 94 | select/3, 95 | select/4, 96 | slot/3, 97 | update_counter/4]). 98 | 99 | %% Index consistency 100 | -export([index_is_consistent/3, 101 | is_index_consistent/2]). 102 | 103 | %% record and key validation 104 | -export([validate_key/6, 105 | validate_record/6]). 106 | 107 | %% file extension callbacks 108 | -export([real_suffixes/0, 109 | tmp_suffixes/0]). 110 | 111 | %% 112 | %% GEN SERVER CALLBACKS AND CALLS 113 | %% 114 | 115 | -export([start_proc/5, 116 | init/1, 117 | handle_call/3, 118 | handle_info/2, 119 | handle_cast/2, 120 | terminate/2, 121 | code_change/3]). 122 | 123 | -export([ix_prefixes/3]). 124 | 125 | 126 | %% ---------------------------------------------------------------------------- 127 | %% DEFINES 128 | %% ---------------------------------------------------------------------------- 129 | 130 | %% Name of the LevelDB interface module; defaults to eleveldb but can be 131 | %% configured by passing -DLEVELDB_MODULE= to erlc. 132 | -ifdef(LEVELDB_MODULE). 133 | -define(leveldb, ?LEVELDB_MODULE). 134 | -else. 135 | -define(leveldb, eleveldb). %% Name of the LevelDB interface module 136 | -endif. 137 | 138 | %% Data and meta data (a.k.a. info) are stored in the same table. 139 | %% This is a table of the first byte in data 140 | %% 0 = before meta data 141 | %% 1 = meta data 142 | %% 2 = before data 143 | %% >= 8 = data 144 | 145 | -define(INFO_START, 0). 146 | -define(INFO_TAG, 1). 147 | -define(DATA_START, 2). 148 | -define(BAG_CNT, 32). % Number of bits used for bag object counter 149 | -define(MAX_BAG, 16#FFFFFFFF). 150 | 151 | %% enable debugging messages through mnesia:set_debug_level(debug) 152 | -ifndef(MNESIA_ELEVELDB_NO_DBG). 153 | -define(dbg(Fmt, Args), 154 | %% avoid evaluating Args if the message will be dropped anyway 155 | case mnesia_monitor:get_env(debug) of 156 | none -> ok; 157 | verbose -> ok; 158 | _ -> mnesia_lib:dbg_out("~p:~p: "++(Fmt),[?MODULE,?LINE|Args]) 159 | end). 160 | -else. 161 | -define(dbg(Fmt, Args), ok). 162 | -endif. 163 | 164 | %% ---------------------------------------------------------------------------- 165 | %% RECORDS 166 | %% ---------------------------------------------------------------------------- 167 | 168 | -record(sel, {alias, % TODO: not used 169 | tab, 170 | ref, 171 | keypat, 172 | record_name, 173 | compiled_ms, 174 | limit, 175 | key_only = false, % TODO: not used, see note below 176 | direction = forward}). % TODO: not used 177 | 178 | %% The optimization for `key_only' iterators has been removed because 179 | %% (1) it was not used, and (2) would need to be rewritten to interact 180 | %% better with dialyzer. 181 | %% 182 | %% Details: ets:match_spec_compile/1 cannot be relied on to crash on 183 | %% inputs which violate its contract (in particular, the function only 184 | %% checks that the match spec is a list of 3-tuples, but performs no 185 | %% checks on the match head). As a result, dialyzer will assume that 186 | %% if ets:match_spec_compile/1 returns successfully, the match spec 187 | %% passed to it fulfills the spec. The original `key_only' 188 | %% optimization had dead code warnings doing runtime type checks which 189 | %% dialyzer deemed unnecessary (because the match spec had been passed 190 | %% to ets:match_spec_compile/1 without crashing). These would need to 191 | %% be rewritten to use try-catch instead to avoid the dead code 192 | %% warnings, but since the optimization isn't being used there was not 193 | %% much point in doing that refactoring. 194 | 195 | -record(st, { ets 196 | , ref 197 | , alias 198 | , tab 199 | , type 200 | , size_warnings % integer() 201 | , record_name % atom() 202 | , maintain_size % boolean() 203 | }). 204 | 205 | %% ---------------------------------------------------------------------------- 206 | %% CONVENIENCE API 207 | %% ---------------------------------------------------------------------------- 208 | 209 | register() -> 210 | register(default_alias()). 211 | 212 | register(Alias) -> 213 | Module = ?MODULE, 214 | case mnesia:add_backend_type(Alias, Module) of 215 | {atomic, ok} -> 216 | {ok, Alias}; 217 | {aborted, {backend_type_already_exists, _}} -> 218 | {ok, Alias}; 219 | {aborted, Reason} -> 220 | {error, Reason} 221 | end. 222 | 223 | default_alias() -> 224 | leveldb_copies. 225 | 226 | 227 | %% ---------------------------------------------------------------------------- 228 | %% DEBUG API 229 | %% ---------------------------------------------------------------------------- 230 | 231 | %% A debug function that shows the leveldb table content 232 | show_table(Tab) -> 233 | show_table(default_alias(), Tab). 234 | 235 | show_table(Alias, Tab) -> 236 | show_table(Alias, Tab, 100). 237 | 238 | show_table(Alias, Tab, Limit) -> 239 | {Ref, _Type, RecName} = get_ref(Alias, Tab), 240 | with_iterator(Ref, fun(I) -> i_show_table(I, first, Limit, RecName) end). 241 | 242 | %% PRIVATE 243 | 244 | i_show_table(_, _, 0, _RecName) -> 245 | {error, skipped_some}; 246 | i_show_table(I, Move, Limit, RecName) -> 247 | case ?leveldb:iterator_move(I, Move) of 248 | {ok, EncKey, EncVal} -> 249 | {Type,Val} = 250 | case EncKey of 251 | << ?INFO_TAG, K/binary >> -> 252 | {info,{decode_key(K),decode_val(EncVal)}}; 253 | _ -> 254 | K = decode_key(EncKey), 255 | V = decode_val(EncVal), 256 | V2 = set_record(2, V, K, RecName), 257 | {data,V2} 258 | end, 259 | io:fwrite("~p: ~p~n", [Type, Val]), 260 | i_show_table(I, next, Limit-1, RecName); 261 | _ -> 262 | ok 263 | end. 264 | 265 | 266 | %% ---------------------------------------------------------------------------- 267 | %% BACKEND CALLBACKS 268 | %% ---------------------------------------------------------------------------- 269 | 270 | %% backend management 271 | 272 | init_backend() -> 273 | stick_eleveldb_dir(), 274 | application:start(mnesia_eleveldb), 275 | ok. 276 | 277 | %% Prevent reloading of modules in eleveldb itself during runtime, since it 278 | %% can lead to inconsistent state in eleveldb and silent data corruption. 279 | stick_eleveldb_dir() -> 280 | case code:which(eleveldb) of 281 | BeamPath when is_list(BeamPath), BeamPath =/= "" -> 282 | Dir = filename:dirname(BeamPath), 283 | case code:stick_dir(Dir) of 284 | ok -> ok; 285 | error -> warn_stick_dir({error, Dir}) 286 | end; 287 | Other -> 288 | warn_stick_dir({not_found, Other}) 289 | end. 290 | 291 | warn_stick_dir(Reason) -> 292 | mnesia_lib:warning("cannot make eleveldb directory sticky:~n~p~n", 293 | [Reason]). 294 | 295 | add_aliases(_Aliases) -> 296 | ok. 297 | 298 | remove_aliases(_Aliases) -> 299 | ok. 300 | 301 | %% schema level callbacks 302 | 303 | %% This function is used to determine what the plugin supports 304 | %% semantics(Alias, storage) -> 305 | %% ram_copies | disc_copies | disc_only_copies (mandatory) 306 | %% semantics(Alias, types) -> 307 | %% [bag | set | ordered_set] (mandatory) 308 | %% semantics(Alias, index_fun) -> 309 | %% fun(Alias, Tab, Pos, Obj) -> [IxValue] (optional) 310 | %% semantics(Alias, _) -> 311 | %% undefined. 312 | %% 313 | semantics(_Alias, storage) -> disc_only_copies; 314 | semantics(_Alias, types ) -> [set, ordered_set, bag]; 315 | semantics(_Alias, index_types) -> [ordered]; 316 | semantics(_Alias, index_fun) -> fun index_f/4; 317 | semantics(_Alias, _) -> undefined. 318 | 319 | is_index_consistent(Alias, {Tab, index, PosInfo}) -> 320 | case info(Alias, Tab, {index_consistent, PosInfo}) of 321 | true -> true; 322 | _ -> false 323 | end. 324 | 325 | index_is_consistent(Alias, {Tab, index, PosInfo}, Bool) 326 | when is_boolean(Bool) -> 327 | write_info(Alias, Tab, {index_consistent, PosInfo}, Bool). 328 | 329 | 330 | %% PRIVATE FUN 331 | index_f(_Alias, _Tab, Pos, Obj) -> 332 | [element(Pos, Obj)]. 333 | 334 | ix_prefixes(_Tab, _Pos, Obj) -> 335 | lists:foldl( 336 | fun(V, Acc) when is_list(V) -> 337 | try Pfxs = prefixes(list_to_binary(V)), 338 | Pfxs ++ Acc 339 | catch 340 | error:_ -> 341 | Acc 342 | end; 343 | (V, Acc) when is_binary(V) -> 344 | Pfxs = prefixes(V), 345 | Pfxs ++ Acc; 346 | (_, Acc) -> 347 | Acc 348 | end, [], tl(tuple_to_list(Obj))). 349 | 350 | prefixes(<>) -> 351 | [P]; 352 | prefixes(_) -> 353 | []. 354 | 355 | %% For now, only verify that the type is set or ordered_set. 356 | %% set is OK as ordered_set is a kind of set. 357 | check_definition(Alias, Tab, Nodes, Props) -> 358 | Id = {Alias, Nodes}, 359 | Props1 = lists:map( 360 | fun({type, T} = P) -> 361 | if T==set; T==ordered_set; T==bag -> 362 | P; 363 | true -> 364 | mnesia:abort({combine_error, 365 | Tab, 366 | [Id, {type,T}]}) 367 | end; 368 | ({user_properties, _} = P) -> 369 | %% should perhaps verify eleveldb options... 370 | P; 371 | (P) -> P 372 | end, Props), 373 | {ok, Props1}. 374 | 375 | %% -> ok | {error, exists} 376 | create_table(_Alias, Tab, _Props) -> 377 | create_mountpoint(Tab). 378 | 379 | load_table(Alias, Tab, _LoadReason, Opts) -> 380 | Type = proplists:get_value(type, Opts), 381 | LdbUserProps = proplists:get_value( 382 | leveldb_opts, proplists:get_value( 383 | user_properties, Opts, []), []), 384 | StorageProps = proplists:get_value( 385 | leveldb, proplists:get_value( 386 | storage_properties, Opts, []), LdbUserProps), 387 | LdbOpts = mnesia_eleveldb_params:lookup(Tab, StorageProps), 388 | ProcName = proc_name(Alias, Tab), 389 | case whereis(ProcName) of 390 | undefined -> 391 | RecName = record_name(Tab, Opts), 392 | load_table_(Alias, Tab, Type, LdbOpts, RecName); 393 | Pid -> 394 | gen_server:call(Pid, {load, Alias, Tab, Type, LdbOpts}, infinity) 395 | end. 396 | 397 | %% In most cases Opts = mnesia_schema:cs2list(Cs), and so will include 398 | %% a {record_name, RecName} entry. However, in mnesia_dumper Opts is a 399 | %% TabDef that allows mnesia_schema:list2cs(TabDef), and the record_name 400 | %% may be absent causing RecName to default to Tab -- we do the same. 401 | record_name(Tab, Opts) -> 402 | case proplists:lookup(record_name, Opts) of 403 | {record_name, RecName} -> RecName; 404 | none -> Tab 405 | end. 406 | 407 | load_table_(Alias, Tab, Type, LdbOpts, RecName) -> 408 | ShutdownTime = proplists:get_value( 409 | owner_shutdown_time, LdbOpts, 120000), 410 | case mnesia_ext_sup:start_proc( 411 | Tab, ?MODULE, start_proc, [Alias,Tab,Type,LdbOpts,RecName], 412 | [{shutdown, ShutdownTime}]) of 413 | {ok, _Pid} -> 414 | ok; 415 | 416 | %% TODO: This reply is according to the manual, but we dont get it. 417 | {error, {already_started, _Pid}} -> 418 | %% TODO: Is it an error if the table already is 419 | %% loaded. This printout is triggered when running 420 | %% transform_table on a leveldb_table that has indexing. 421 | ?dbg("ERR: table:~p already loaded pid:~p~n", 422 | [Tab, _Pid]), 423 | ok; 424 | 425 | %% TODO: This reply is not according to the manual, but we get it. 426 | {error, {{already_started, _Pid}, _Stack}} -> 427 | %% TODO: Is it an error if the table already is 428 | %% loaded. This printout is triggered when running 429 | %% transform_table on a leveldb_table that has indexing. 430 | ?dbg("ERR: table:~p already loaded pid:~p stack:~p~n", 431 | [Tab, _Pid, _Stack]), 432 | ok 433 | end. 434 | 435 | close_table(Alias, Tab) -> 436 | ?dbg("~p: close_table(~p, ~p);~n Trace: ~s~n", 437 | [self(), Alias, Tab, pp_stack()]), 438 | if is_atom(Tab) -> 439 | [close_table(Alias, R) 440 | || {R, _} <- related_resources(Tab)]; 441 | true -> 442 | ok 443 | end, 444 | close_table_(Alias, Tab). 445 | 446 | close_table_(Alias, Tab) -> 447 | case opt_call(Alias, Tab, close_table) of 448 | {error, noproc} -> 449 | ?dbg("~p: close_table_(~p) -> noproc~n", 450 | [self(), Tab]), 451 | ok; 452 | {ok, _} -> 453 | ok 454 | end. 455 | 456 | -ifndef(MNESIA_ELEVELDB_NO_DBG). 457 | pp_stack() -> 458 | Trace = try throw(true) 459 | catch 460 | _:_:StackTrace -> 461 | case StackTrace of 462 | [_|T] -> T; 463 | [] -> [] 464 | end 465 | end, 466 | pp_calls(10, Trace). 467 | 468 | pp_calls(I, [{M,F,A,Pos} | T]) -> 469 | Spc = lists:duplicate(I, $\s), 470 | Pp = fun(Mx,Fx,Ax,Px) -> 471 | [atom_to_list(Mx),":",atom_to_list(Fx),"/",integer_to_list(Ax), 472 | pp_pos(Px)] 473 | end, 474 | [Pp(M,F,A,Pos)|[["\n",Spc,Pp(M1,F1,A1,P1)] || {M1,F1,A1,P1} <- T]]. 475 | 476 | pp_pos([]) -> ""; 477 | pp_pos([{file,_},{line,L}]) -> 478 | [" (", integer_to_list(L), ")"]. 479 | 480 | -endif. 481 | 482 | sync_close_table(Alias, Tab) -> 483 | ?dbg("~p: sync_close_table(~p, ~p);~n Trace: ~s~n", 484 | [self(), Alias, Tab, pp_stack()]), 485 | close_table(Alias, Tab). 486 | 487 | delete_table(Alias, Tab) -> 488 | ?dbg("~p: delete_table(~p, ~p);~n Trace: ~s~n", 489 | [self(), Alias, Tab, pp_stack()]), 490 | delete_table(Alias, Tab, data_mountpoint(Tab)). 491 | 492 | delete_table(Alias, Tab, MP) -> 493 | if is_atom(Tab) -> 494 | [delete_table(Alias, T, M) || {T,M} <- related_resources(Tab)]; 495 | true -> 496 | ok 497 | end, 498 | case opt_call(Alias, Tab, delete_table) of 499 | {error, noproc} -> 500 | do_delete_table(Tab, MP); 501 | {ok, _} -> 502 | ok 503 | end. 504 | 505 | do_delete_table(Tab, MP) -> 506 | assert_proper_mountpoint(Tab, MP), 507 | destroy_db(MP, []). 508 | 509 | 510 | info(_Alias, Tab, memory) -> 511 | try ets:info(tab_name(icache, Tab), memory) 512 | catch 513 | error:_ -> 514 | 0 515 | end; 516 | info(Alias, Tab, size) -> 517 | case retrieve_size(Alias, Tab) of 518 | {ok, Size} -> 519 | if Size < 10000 -> ok; 520 | true -> size_warning(Alias, Tab) 521 | end, 522 | Size; 523 | Error -> 524 | Error 525 | end; 526 | info(_Alias, Tab, Item) -> 527 | case try_read_info(Tab, Item, undefined) of 528 | {ok, Value} -> 529 | Value; 530 | Error -> 531 | Error 532 | end. 533 | 534 | retrieve_size(_Alias, Tab) -> 535 | case try_read_info(Tab, size, 0) of 536 | {ok, Size} -> 537 | {ok, Size}; 538 | Error -> 539 | Error 540 | end. 541 | 542 | try_read_info(Tab, Item, Default) -> 543 | try 544 | {ok, read_info(Item, Default, tab_name(icache, Tab))} 545 | catch 546 | error:Reason -> 547 | {error, Reason} 548 | end. 549 | 550 | write_info(Alias, Tab, Key, Value) -> 551 | call(Alias, Tab, {write_info, Key, Value}). 552 | 553 | %% table synch calls 554 | 555 | %% =========================================================== 556 | %% Table synch protocol 557 | %% Callbacks are 558 | %% Sender side: 559 | %% 1. sender_init(Alias, Tab, RemoteStorage, ReceiverPid) -> 560 | %% {standard, InitFun, ChunkFun} | {InitFun, ChunkFun} when 561 | %% InitFun :: fun() -> {Recs, Cont} | '$end_of_table' 562 | %% ChunkFun :: fun(Cont) -> {Recs, Cont1} | '$end_of_table' 563 | %% 564 | %% If {standard, I, C} is returned, the standard init message will be 565 | %% sent to the receiver. Matching on RemoteStorage can reveal if a 566 | %% different protocol can be used. 567 | %% 568 | %% 2. InitFun() is called 569 | %% 3a. ChunkFun(Cont) is called repeatedly until done 570 | %% 3b. sender_handle_info(Msg, Alias, Tab, ReceiverPid, Cont) -> 571 | %% {ChunkFun, NewCont} 572 | %% 573 | %% Receiver side: 574 | %% 1. receiver_first_message(SenderPid, Msg, Alias, Tab) -> 575 | %% {Size::integer(), State} 576 | %% 2. receive_data(Data, Alias, Tab, _Sender, State) -> 577 | %% {more, NewState} | {{more, Msg}, NewState} 578 | %% 3. receive_done(_Alias, _Tab, _Sender, _State) -> 579 | %% ok 580 | %% 581 | %% The receiver can communicate with the Sender by returning 582 | %% {{more, Msg}, St} from receive_data/4. The sender will be called through 583 | %% sender_handle_info(Msg, ...), where it can adjust its ChunkFun and 584 | %% Continuation. Note that the message from the receiver is sent once the 585 | %% receive_data/4 function returns. This is slightly different from the 586 | %% normal mnesia table synch, where the receiver acks immediately upon 587 | %% reception of a new chunk, then processes the data. 588 | %% 589 | 590 | sender_init(Alias, Tab, _RemoteStorage, _Pid) -> 591 | %% Need to send a message to the receiver. It will be handled in 592 | %% receiver_first_message/4 below. There could be a volley of messages... 593 | {standard, 594 | fun() -> 595 | select(Alias, Tab, [{'_',[],['$_']}], 100) 596 | end, 597 | chunk_fun()}. 598 | 599 | sender_handle_info(_Msg, _Alias, _Tab, _ReceiverPid, Cont) -> 600 | %% ignore - we don't expect any message from the receiver 601 | {chunk_fun(), Cont}. 602 | 603 | receiver_first_message(_Pid, {first, Size} = _Msg, _Alias, _Tab) -> 604 | {Size, _State = []}. 605 | 606 | receive_data(Data, Alias, Tab, _Sender, State) -> 607 | [insert(Alias, Tab, Obj) || Obj <- Data], 608 | {more, State}. 609 | 610 | receive_done(_Alias, _Tab, _Sender, _State) -> 611 | ok. 612 | 613 | %% End of table synch protocol 614 | %% =========================================================== 615 | 616 | %% PRIVATE 617 | 618 | chunk_fun() -> 619 | fun(Cont) -> 620 | select(Cont) 621 | end. 622 | 623 | %% low-level accessor callbacks. 624 | 625 | delete(Alias, Tab, Key) -> 626 | opt_call(Alias, Tab, {delete, encode_key(Key)}), 627 | ok. 628 | 629 | first(Alias, Tab) -> 630 | {Ref, _Type, _RecName} = get_ref(Alias, Tab), 631 | with_keys_only_iterator(Ref, fun i_first/1). 632 | 633 | %% PRIVATE ITERATOR 634 | i_first(I) -> 635 | case ?leveldb:iterator_move(I, <>) of 636 | {ok, First} -> 637 | decode_key(First); 638 | _ -> 639 | '$end_of_table' 640 | end. 641 | 642 | %% Not relevant for an ordered_set 643 | fixtable(_Alias, _Tab, _Bool) -> 644 | true. 645 | 646 | %% To save storage space, we avoid storing the key twice. We replace the key 647 | %% in the record with []. It has to be put back in lookup/3. Similarly we 648 | %% avoid storing the record name. 649 | insert(Alias, Tab, Obj) -> 650 | Pos = keypos(Tab), 651 | EncKey = encode_key(element(Pos, Obj)), 652 | EncVal = encode_val(set_record(Pos, Obj, [], [])), 653 | call(Alias, Tab, {insert, EncKey, EncVal}). 654 | 655 | last(Alias, Tab) -> 656 | {Ref, _Type, _RecName} = get_ref(Alias, Tab), 657 | with_keys_only_iterator(Ref, fun i_last/1). 658 | 659 | %% PRIVATE ITERATOR 660 | i_last(I) -> 661 | case ?leveldb:iterator_move(I, last) of 662 | {ok, << ?INFO_TAG, _/binary >>} -> 663 | '$end_of_table'; 664 | {ok, Last} -> 665 | decode_key(Last); 666 | _ -> 667 | '$end_of_table' 668 | end. 669 | 670 | %% Since we replace the key with [] in the record, we have to put it back 671 | %% into the found record. 672 | lookup(Alias, Tab, Key) -> 673 | Enc = encode_key(Key), 674 | {Ref, Type, RecName} = get_ref(Alias, Tab), 675 | case Type of 676 | bag -> lookup_bag(Ref, Key, Enc, keypos(Tab), RecName); 677 | _ -> 678 | case ?leveldb:get(Ref, Enc, []) of 679 | {ok, EncVal} -> 680 | [set_record(keypos(Tab), decode_val(EncVal), Key, RecName)]; 681 | _ -> 682 | [] 683 | end 684 | end. 685 | 686 | lookup_bag(Ref, K, Enc, KP, RecName) -> 687 | Sz = byte_size(Enc), 688 | with_iterator( 689 | Ref, fun(I) -> 690 | lookup_bag_(Sz, Enc, ?leveldb:iterator_move(I, Enc), 691 | K, I, KP, RecName) 692 | end). 693 | 694 | lookup_bag_(Sz, Enc, {ok, Enc, _}, K, I, KP, RecName) -> 695 | lookup_bag_(Sz, Enc, ?leveldb:iterator_move(I, next), K, I, KP, RecName); 696 | lookup_bag_(Sz, Enc, Res, K, I, KP, RecName) -> 697 | case Res of 698 | {ok, <>, V} -> 699 | [set_record(KP, decode_val(V), K, RecName) | 700 | lookup_bag_(Sz, Enc, ?leveldb:iterator_move(I, next), K, I, KP, RecName)]; 701 | _ -> 702 | [] 703 | end. 704 | 705 | match_delete(Alias, Tab, Pat) when is_atom(Pat) -> 706 | %do_match_delete(Alias, Tab, '_'), 707 | case is_wild(Pat) of 708 | true -> 709 | call(Alias, Tab, clear_table), 710 | ok; 711 | false -> 712 | %% can this happen?? 713 | error(badarg) 714 | end; 715 | match_delete(Alias, Tab, Pat) when is_tuple(Pat) -> 716 | KP = keypos(Tab), 717 | Key = element(KP, Pat), 718 | case is_wild(Key) of 719 | true -> 720 | call(Alias, Tab, clear_table); 721 | false -> 722 | call(Alias, Tab, {match_delete, Pat}) 723 | end, 724 | ok. 725 | 726 | 727 | next(Alias, Tab, Key) -> 728 | {Ref, _Type, _RecName} = get_ref(Alias, Tab), 729 | EncKey = encode_key(Key), 730 | with_keys_only_iterator(Ref, fun(I) -> i_next(I, EncKey, Key) end). 731 | 732 | %% PRIVATE ITERATOR 733 | i_next(I, EncKey, Key) -> 734 | case ?leveldb:iterator_move(I, EncKey) of 735 | {ok, EncKey} -> 736 | i_next_loop(?leveldb:iterator_move(I, next), I, Key); 737 | Other -> 738 | i_next_loop(Other, I, Key) 739 | end. 740 | 741 | i_next_loop({ok, EncKey}, I, Key) -> 742 | case decode_key(EncKey) of 743 | Key -> 744 | i_next_loop(?leveldb:iterator_move(I, next), I, Key); 745 | NextKey -> 746 | NextKey 747 | end; 748 | i_next_loop(_, _I, _Key) -> 749 | '$end_of_table'. 750 | 751 | prev(Alias, Tab, Key0) -> 752 | {Ref, _Type, _RecName} = get_ref(Alias, Tab), 753 | Key = encode_key(Key0), 754 | with_keys_only_iterator(Ref, fun(I) -> i_prev(I, Key) end). 755 | 756 | %% PRIVATE ITERATOR 757 | i_prev(I, Key) -> 758 | case ?leveldb:iterator_move(I, Key) of 759 | {ok, _} -> 760 | i_move_to_prev(I, Key); 761 | {error, invalid_iterator} -> 762 | i_last(I) 763 | end. 764 | 765 | %% PRIVATE ITERATOR 766 | i_move_to_prev(I, Key) -> 767 | case ?leveldb:iterator_move(I, prev) of 768 | {ok, << ?INFO_TAG, _/binary >>} -> 769 | '$end_of_table'; 770 | {ok, Prev} when Prev < Key -> 771 | decode_key(Prev); 772 | {ok, _} -> 773 | i_move_to_prev(I, Key); 774 | _ -> 775 | '$end_of_table' 776 | end. 777 | 778 | repair_continuation(Cont, _Ms) -> 779 | Cont. 780 | 781 | select(Cont) -> 782 | %% Handle {ModOrAlias, Cont} wrappers for backwards compatibility with 783 | %% older versions of mnesia_ext (before OTP 20). 784 | case Cont of 785 | {_, '$end_of_table'} -> '$end_of_table'; 786 | {_, Cont1} -> Cont1(); 787 | '$end_of_table' -> '$end_of_table'; 788 | _ -> Cont() 789 | end. 790 | 791 | select(Alias, Tab, Ms) -> 792 | case select(Alias, Tab, Ms, infinity) of 793 | {Res, '$end_of_table'} -> 794 | Res; 795 | '$end_of_table' -> 796 | '$end_of_table' 797 | end. 798 | 799 | select(_Alias, _Tab, _Ms = [], _Limit) -> 800 | {[], '$end_of_table'}; 801 | select(Alias, Tab, Ms, Limit) when Limit==infinity; is_integer(Limit) -> 802 | {Ref, Type, RecName} = get_ref(Alias, Tab), 803 | do_select(Ref, Tab, Type, Ms, Limit, RecName). 804 | 805 | slot(Alias, Tab, Pos) when is_integer(Pos), Pos >= 0 -> 806 | {Ref, Type, RecName} = get_ref(Alias, Tab), 807 | First = fun(I) -> ?leveldb:iterator_move(I, <>) end, 808 | F = case Type of 809 | bag -> fun(I) -> slot_iter_set(First(I), I, 0, Pos, RecName) end; 810 | _ -> fun(I) -> slot_iter_set(First(I), I, 0, Pos, RecName) end 811 | end, 812 | with_iterator(Ref, F); 813 | slot(_, _, _) -> 814 | error(badarg). 815 | 816 | %% Exactly which objects Mod:slot/2 is supposed to return is not defined, 817 | %% so let's just use the same version for both set and bag. No one should 818 | %% use this function anyway, as it is ridiculously inefficient. 819 | slot_iter_set({ok, K, V}, _I, P, P, RecName) -> 820 | [set_record(2, decode_val(V), decode_key(K), RecName)]; 821 | slot_iter_set({ok, _, _}, I, P1, P, RecName) when P1 < P -> 822 | slot_iter_set(?leveldb:iterator_move(I, next), I, P1+1, P, RecName); 823 | slot_iter_set(Res, _, _, _, _) when element(1, Res) =/= ok -> 824 | '$end_of_table'. 825 | 826 | update_counter(Alias, Tab, C, Val) when is_integer(Val) -> 827 | case call(Alias, Tab, {update_counter, C, Val}) of 828 | badarg -> 829 | mnesia:abort(badarg); 830 | Res -> 831 | Res 832 | end. 833 | 834 | %% server-side part 835 | do_update_counter(C, Val, Ref) -> 836 | Enc = encode_key(C), 837 | case ?leveldb:get(Ref, Enc, [{fill_cache, true}]) of 838 | {ok, EncVal} -> 839 | case decode_val(EncVal) of 840 | {_, _, Old} = Rec when is_integer(Old) -> 841 | Res = Old+Val, 842 | ?leveldb:put(Ref, Enc, 843 | encode_val( 844 | setelement(3, Rec, Res)), 845 | []), 846 | Res; 847 | _ -> 848 | badarg 849 | end; 850 | _ -> 851 | badarg 852 | end. 853 | 854 | %% PRIVATE 855 | 856 | %% key+data iterator: iterator_move/2 returns {ok, EncKey, EncVal} 857 | with_iterator(Ref, F) -> 858 | {ok, I} = ?leveldb:iterator(Ref, []), 859 | try F(I) 860 | after 861 | ?leveldb:iterator_close(I) 862 | end. 863 | 864 | %% keys_only iterator: iterator_move/2 returns {ok, EncKey} 865 | with_keys_only_iterator(Ref, F) -> 866 | {ok, I} = ?leveldb:iterator(Ref, [], keys_only), 867 | try F(I) 868 | after 869 | ?leveldb:iterator_close(I) 870 | end. 871 | 872 | %% TODO - use with_keys_only_iterator for match_delete 873 | 874 | %% record and key validation 875 | 876 | validate_key(_Alias, _Tab, RecName, Arity, Type, _Key) -> 877 | {RecName, Arity, Type}. 878 | 879 | validate_record(_Alias, _Tab, RecName, Arity, Type, _Obj) -> 880 | {RecName, Arity, Type}. 881 | 882 | %% file extension callbacks 883 | 884 | %% Extensions for files that are permanent. Needs to be cleaned up 885 | %% e.g. at deleting the schema. 886 | real_suffixes() -> 887 | [".extldb"]. 888 | 889 | %% Extensions for temporary files. Can be cleaned up when mnesia 890 | %% cleans up other temporary files. 891 | tmp_suffixes() -> 892 | []. 893 | 894 | 895 | %% ---------------------------------------------------------------------------- 896 | %% GEN SERVER CALLBACKS AND CALLS 897 | %% ---------------------------------------------------------------------------- 898 | 899 | start_proc(Alias, Tab, Type, LdbOpts, RecName) -> 900 | ProcName = proc_name(Alias, Tab), 901 | gen_server:start_link({local, ProcName}, ?MODULE, 902 | {Alias, Tab, Type, LdbOpts, RecName}, []). 903 | 904 | init({Alias, Tab, Type, LdbOpts, RecName}) -> 905 | process_flag(trap_exit, true), 906 | {ok, Ref, Ets} = do_load_table(Tab, LdbOpts), 907 | St = #st{ ets = Ets 908 | , ref = Ref 909 | , alias = Alias 910 | , tab = Tab 911 | , type = Type 912 | , size_warnings = 0 913 | , record_name = RecName 914 | , maintain_size = should_maintain_size(Tab) 915 | }, 916 | {ok, recover_size_info(St)}. 917 | 918 | do_load_table(Tab, LdbOpts) -> 919 | MPd = data_mountpoint(Tab), 920 | ?dbg("** Mountpoint: ~p~n ~s~n", [MPd, os:cmd("ls " ++ MPd)]), 921 | Ets = ets:new(tab_name(icache,Tab), [set, protected, named_table]), 922 | {ok, Ref} = open_leveldb(MPd, LdbOpts), 923 | leveldb_to_ets(Ref, Ets), 924 | {ok, Ref, Ets}. 925 | 926 | handle_call({load, Alias, Tab, Type, LdbOpts}, _From, 927 | #st{type = Type, alias = Alias, tab = Tab} = St) -> 928 | {ok, Ref, Ets} = do_load_table(Tab, LdbOpts), 929 | {reply, ok, St#st{ref = Ref, ets = Ets}}; 930 | handle_call(get_ref, _From, #st{ref = Ref, type = Type, record_name = RecName} = St) -> 931 | {reply, {Ref, Type, RecName}, St}; 932 | handle_call({write_info, Key, Value}, _From, #st{} = St) -> 933 | _ = write_info_(Key, Value, St), 934 | {reply, ok, St}; 935 | handle_call({update_counter, C, Incr}, _From, #st{ref = Ref} = St) -> 936 | {reply, do_update_counter(C, Incr, Ref), St}; 937 | handle_call({insert, Key, Val}, _From, St) -> 938 | do_insert(Key, Val, St), 939 | {reply, ok, St}; 940 | handle_call({delete, Key}, _From, St) -> 941 | do_delete(Key, St), 942 | {reply, ok, St}; 943 | handle_call(clear_table, _From, #st{ets = Ets, tab = Tab, ref = Ref} = St) -> 944 | MPd = data_mountpoint(Tab), 945 | ?dbg("Attempting clear_table(~p)~n", [Tab]), 946 | _ = eleveldb_close(Ref), 947 | {ok, NewRef} = destroy_recreate(MPd, leveldb_open_opts(Tab)), 948 | ets:delete_all_objects(Ets), 949 | leveldb_to_ets(NewRef, Ets), 950 | {reply, ok, St#st{ref = NewRef}}; 951 | handle_call({match_delete, Pat}, _From, #st{} = St) -> 952 | Res = do_match_delete(Pat, St), 953 | {reply, Res, St}; 954 | handle_call(close_table, _From, #st{ref = Ref, ets = Ets} = St) -> 955 | _ = eleveldb_close(Ref), 956 | ets:delete(Ets), 957 | {reply, ok, St#st{ref = undefined}}; 958 | handle_call(delete_table, _From, #st{tab = T, ref = Ref, ets = Ets} = St) -> 959 | _ = (catch eleveldb_close(Ref)), 960 | _ = (catch ets:delete(Ets)), 961 | do_delete_table(T, data_mountpoint(T)), 962 | {stop, normal, ok, St#st{ref = undefined}}. 963 | 964 | handle_cast(size_warning, #st{tab = T, size_warnings = W} = St) when W < 10 -> 965 | mnesia_lib:warning("large size retrieved from table: ~p~n", [T]), 966 | if W =:= 9 -> 967 | OneHrMs = 60 * 60 * 1000, 968 | erlang:send_after(OneHrMs, self(), unmute_size_warnings); 969 | true -> 970 | ok 971 | end, 972 | {noreply, St#st{size_warnings = W + 1}}; 973 | handle_cast(size_warning, #st{size_warnings = W} = St) when W >= 10 -> 974 | {noreply, St#st{size_warnings = W + 1}}; 975 | handle_cast(_, St) -> 976 | {noreply, St}. 977 | 978 | handle_info(unmute_size_warnings, #st{tab = T, size_warnings = W} = St) -> 979 | C = W - 10, 980 | if C > 0 -> 981 | mnesia_lib:warning("warnings suppressed~ntable: ~p, count: ~p~n", 982 | [T, C]); 983 | true -> 984 | ok 985 | end, 986 | {noreply, St#st{size_warnings = 0}}; 987 | handle_info({'EXIT', _, _} = _EXIT, St) -> 988 | ?dbg("leveldb owner received ~p~n", [_EXIT]), 989 | {noreply, St}; 990 | handle_info(_, St) -> 991 | {noreply, St}. 992 | 993 | code_change(_FromVsn, St, _Extra) -> 994 | {ok, St}. 995 | 996 | terminate(_Reason, #st{ref = Ref}) -> 997 | if Ref =/= undefined -> 998 | ?leveldb:close(Ref); 999 | true -> ok 1000 | end, 1001 | ok. 1002 | 1003 | 1004 | %% ---------------------------------------------------------------------------- 1005 | %% GEN SERVER PRIVATE 1006 | %% ---------------------------------------------------------------------------- 1007 | 1008 | get_env_default(Key, Default) -> 1009 | case os:getenv(Key) of 1010 | false -> 1011 | Default; 1012 | Value -> 1013 | Value 1014 | end. 1015 | 1016 | leveldb_open_opts({Tab, index, {Pos,_}}) -> 1017 | UserProps = mnesia_lib:val({Tab, user_properties}), 1018 | IxOpts = proplists:get_value(leveldb_index_opts, UserProps, []), 1019 | PosOpts = proplists:get_value(Pos, IxOpts, []), 1020 | leveldb_open_opts_(PosOpts); 1021 | leveldb_open_opts(Tab) -> 1022 | UserProps = mnesia_lib:val({Tab, user_properties}), 1023 | LdbOpts = proplists:get_value(leveldb_opts, UserProps, []), 1024 | leveldb_open_opts_(LdbOpts). 1025 | 1026 | leveldb_open_opts_(LdbOpts) -> 1027 | lists:foldl( 1028 | fun({K,_} = Item, Acc) -> 1029 | lists:keystore(K, 1, Acc, Item) 1030 | end, default_open_opts(), LdbOpts). 1031 | 1032 | default_open_opts() -> 1033 | [ {create_if_missing, true} 1034 | , {cache_size, 1035 | list_to_integer(get_env_default("LEVELDB_CACHE_SIZE", "32212254"))} 1036 | , {block_size, 1024} 1037 | , {max_open_files, 100} 1038 | , {write_buffer_size, 1039 | list_to_integer(get_env_default( 1040 | "LEVELDB_WRITE_BUFFER_SIZE", "4194304"))} 1041 | , {compression, 1042 | list_to_atom(get_env_default("LEVELDB_COMPRESSION", "true"))} 1043 | , {use_bloomfilter, true} 1044 | ]. 1045 | 1046 | destroy_recreate(MPd, LdbOpts) -> 1047 | ok = destroy_db(MPd, []), 1048 | open_leveldb(MPd, LdbOpts). 1049 | 1050 | open_leveldb(MPd, LdbOpts) -> 1051 | open_leveldb(MPd, leveldb_open_opts_(LdbOpts), get_retries()). 1052 | 1053 | %% Code adapted from basho/riak_kv_eleveldb_backend.erl 1054 | open_leveldb(MPd, Opts, Retries) -> 1055 | open_db(MPd, Opts, max(1, Retries), undefined). 1056 | 1057 | open_db(_, _, 0, LastError) -> 1058 | {error, LastError}; 1059 | open_db(MPd, Opts, RetriesLeft, _) -> 1060 | case ?leveldb:open(MPd, Opts) of 1061 | {ok, Ref} -> 1062 | ?dbg("~p: Open - Leveldb: ~s~n -> {ok, ~p}~n", 1063 | [self(), MPd, Ref]), 1064 | {ok, Ref}; 1065 | %% Check specifically for lock error, this can be caused if 1066 | %% a crashed mnesia takes some time to flush leveldb information 1067 | %% out to disk. The process is gone, but the NIF resource cleanup 1068 | %% may not have completed. 1069 | {error, {db_open, OpenErr}=Reason} -> 1070 | case lists:prefix("IO error: lock ", OpenErr) of 1071 | true -> 1072 | SleepFor = get_retry_delay(), 1073 | ?dbg("~p: Open - Leveldb backend retrying ~p in ~p ms" 1074 | " after error ~s\n", 1075 | [self(), MPd, SleepFor, OpenErr]), 1076 | timer:sleep(SleepFor), 1077 | open_db(MPd, Opts, RetriesLeft - 1, Reason); 1078 | false -> 1079 | {error, Reason} 1080 | end; 1081 | {error, Reason} -> 1082 | {error, Reason} 1083 | end. 1084 | 1085 | %% await_db_closed(Tab) -> 1086 | %% MPd = data_mountpoint(Tab), 1087 | %% await_db_closed_(MPd). 1088 | 1089 | %% await_db_closed_(MPd) -> 1090 | %% case filelib:is_file(filename:join(MPd, "LOCK")) of 1091 | %% true -> 1092 | %% SleepFor = get_retry_delay(), 1093 | %% timer:sleep(SleepFor), 1094 | %% await_db_closed_(MPd); 1095 | %% false -> 1096 | %% ok 1097 | %% end. 1098 | 1099 | eleveldb_close(undefined) -> 1100 | ok; 1101 | eleveldb_close(Ref) -> 1102 | Res = ?leveldb:close(Ref), 1103 | erlang:garbage_collect(), 1104 | Res. 1105 | 1106 | destroy_db(MPd, Opts) -> 1107 | destroy_db(MPd, Opts, get_retries()). 1108 | 1109 | %% Essentially same code as above. 1110 | destroy_db(MPd, Opts, Retries) -> 1111 | _DRes = destroy_db(MPd, Opts, max(1, Retries), undefined), 1112 | ?dbg("~p: Destroy ~s -> ~p~n", [self(), MPd, _DRes]), 1113 | [_|_] = MPd, % ensure MPd is non-empty 1114 | _RmRes = os:cmd("rm -rf " ++ MPd ++ "/*"), 1115 | ?dbg("~p: RmRes = '~s'~n", [self(), _RmRes]), 1116 | ok. 1117 | 1118 | destroy_db(_, _, 0, LastError) -> 1119 | {error, LastError}; 1120 | destroy_db(MPd, Opts, RetriesLeft, _) -> 1121 | case ?leveldb:destroy(MPd, Opts) of 1122 | ok -> 1123 | ok; 1124 | %% Check specifically for lock error, this can be caused if 1125 | %% destroy follows quickly after close. 1126 | {error, {error_db_destroy, Err}=Reason} -> 1127 | case lists:prefix("IO error: lock ", Err) of 1128 | true -> 1129 | SleepFor = get_retry_delay(), 1130 | ?dbg("~p: Destroy - Leveldb backend retrying ~p in ~p ms" 1131 | " after error ~s\n" 1132 | " children = ~p~n", 1133 | [self(), MPd, SleepFor, Err, 1134 | supervisor:which_children(mnesia_ext_sup)]), 1135 | timer:sleep(SleepFor), 1136 | destroy_db(MPd, Opts, RetriesLeft - 1, Reason); 1137 | false -> 1138 | {error, Reason} 1139 | end; 1140 | {error, Reason} -> 1141 | {error, Reason} 1142 | end. 1143 | 1144 | get_retries() -> 30. 1145 | get_retry_delay() -> 10000. 1146 | 1147 | leveldb_to_ets(Ref, Ets) -> 1148 | with_iterator(Ref, fun(I) -> 1149 | i_leveldb_to_ets(I, Ets, <>) 1150 | end). 1151 | 1152 | i_leveldb_to_ets(I, Ets, Move) -> 1153 | case ?leveldb:iterator_move(I, Move) of 1154 | {ok, << ?INFO_TAG, EncKey/binary >>, EncVal} -> 1155 | Item = decode_key(EncKey), 1156 | Val = decode_val(EncVal), 1157 | ets:insert(Ets, {{info,Item}, Val}), 1158 | i_leveldb_to_ets(I, Ets, next); 1159 | _ -> 1160 | '$end_of_table' 1161 | end. 1162 | 1163 | opt_call(Alias, Tab, Req) -> 1164 | ProcName = proc_name(Alias, Tab), 1165 | case whereis(ProcName) of 1166 | undefined -> 1167 | ?dbg("proc_name(~p, ~p): ~p; NO PROCESS~n", 1168 | [Alias, Tab, ProcName]), 1169 | {error, noproc}; 1170 | Pid when is_pid(Pid) -> 1171 | ?dbg("proc_name(~p, ~p): ~p; Pid = ~p~n", 1172 | [Alias, Tab, ProcName, Pid]), 1173 | {ok, gen_server:call(Pid, Req, infinity)} 1174 | end. 1175 | 1176 | call(Alias, Tab, Req) -> 1177 | ProcName = proc_name(Alias, Tab), 1178 | case gen_server:call(ProcName, Req, infinity) of 1179 | badarg -> 1180 | mnesia:abort(badarg); 1181 | {abort, _} = Err -> 1182 | mnesia:abort(Err); 1183 | Reply -> 1184 | Reply 1185 | end. 1186 | 1187 | size_warning(Alias, Tab) -> 1188 | ProcName = proc_name(Alias, Tab), 1189 | gen_server:cast(ProcName, size_warning). 1190 | 1191 | %% server-side end of insert/3. 1192 | do_insert(K, V, #st{ref = Ref, type = bag, maintain_size = false}) -> 1193 | do_insert_bag(Ref, K, V, false); 1194 | do_insert(K, V, #st{ets = Ets, ref = Ref, type = bag, maintain_size = true}) -> 1195 | CurSz = read_info(size, 0, Ets), 1196 | NewSz = do_insert_bag(Ref, K, V, CurSz), 1197 | ets_insert_info(Ets, size, NewSz), 1198 | ok; 1199 | do_insert(K, V, #st{ref = Ref, maintain_size = false}) -> 1200 | ?leveldb:put(Ref, K, V, []); 1201 | do_insert(K, V, #st{ets = Ets, ref = Ref, maintain_size = true}) -> 1202 | IsNew = 1203 | case ?leveldb:get(Ref, K, []) of 1204 | {ok, _} -> 1205 | false; 1206 | _ -> 1207 | true 1208 | end, 1209 | case IsNew of 1210 | true -> 1211 | NewSz = read_info(size, 0, Ets) + 1, 1212 | {Ki, Vi} = info_obj(size, NewSz), 1213 | ?leveldb:write(Ref, [{put, Ki, Vi}, {put, K, V}], []), 1214 | ets_insert_info(Ets, size, NewSz); 1215 | false -> 1216 | ?leveldb:put(Ref, K, V, []) 1217 | end, 1218 | ok. 1219 | 1220 | do_insert_bag(Ref, K, V, CurSz) -> 1221 | KSz = byte_size(K), 1222 | with_iterator( 1223 | Ref, fun(I) -> 1224 | do_insert_bag_( 1225 | KSz, K, ?leveldb:iterator_move(I, K), I, V, 0, Ref, CurSz) 1226 | end). 1227 | 1228 | 1229 | %% There's a potential access pattern that would force counters to 1230 | %% creep upwards and eventually hit the limit. This could be addressed, 1231 | %% with compaction. TODO. 1232 | do_insert_bag_(Sz, K, Res, I, V, Prev, Ref, TSz) when Prev < ?MAX_BAG -> 1233 | case Res of 1234 | {ok, <>, V} -> 1235 | %% object exists 1236 | TSz; 1237 | {ok, <>, _} -> 1238 | do_insert_bag_( 1239 | Sz, K, ?leveldb:iterator_move(I, next), I, V, N, Ref, TSz); 1240 | _ when TSz =:= false -> 1241 | Key = <>, 1242 | ?leveldb:put(Ref, Key, V, []); 1243 | _ -> 1244 | NewSz = TSz + 1, 1245 | {Ki, Vi} = info_obj(size, NewSz), 1246 | Key = <>, 1247 | ?leveldb:write(Ref, [{put, Ki, Vi}, {put, Key, V}], []), 1248 | NewSz 1249 | end. 1250 | 1251 | %% server-side part 1252 | do_delete(Key, #st{ref = Ref, type = bag, maintain_size = false}) -> 1253 | do_delete_bag(byte_size(Key), Key, Ref, false); 1254 | do_delete(Key, #st{ets = Ets, ref = Ref, type = bag, maintain_size = true}) -> 1255 | Sz = byte_size(Key), 1256 | CurSz = read_info(size, 0, Ets), 1257 | NewSz = do_delete_bag(Sz, Key, Ref, CurSz), 1258 | ets_insert_info(Ets, size, NewSz), 1259 | ok; 1260 | do_delete(Key, #st{ref = Ref, maintain_size = false}) -> 1261 | ?leveldb:delete(Ref, Key, []); 1262 | do_delete(Key, #st{ets = Ets, ref = Ref, maintain_size = true}) -> 1263 | CurSz = read_info(size, 0, Ets), 1264 | case ?leveldb:get(Ref, Key, [{fill_cache,true}]) of 1265 | {ok, _} -> 1266 | NewSz = CurSz -1, 1267 | {Ki, Vi} = info_obj(size, NewSz), 1268 | ok = ?leveldb:write(Ref, [{delete, Key}, {put, Ki, Vi}], []), 1269 | ets_insert_info(Ets, size, NewSz); 1270 | not_found -> 1271 | false 1272 | end. 1273 | 1274 | do_delete_bag(Sz, Key, Ref, TSz) -> 1275 | Found = 1276 | with_keys_only_iterator( 1277 | Ref, fun(I) -> 1278 | do_delete_bag_(Sz, Key, ?leveldb:iterator_move(I, Key), 1279 | Ref, I) 1280 | end), 1281 | case {Found, TSz} of 1282 | {[], _} -> 1283 | TSz; 1284 | {_, false} -> 1285 | ?leveldb:write(Ref, [{delete, K} || K <- Found], []); 1286 | {_, _} -> 1287 | N = length(Found), 1288 | NewSz = TSz - N, 1289 | {Ki, Vi} = info_obj(size, NewSz), 1290 | ?leveldb:write(Ref, [{put, Ki, Vi} | 1291 | [{delete, K} || K <- Found]], []), 1292 | NewSz 1293 | end. 1294 | 1295 | do_delete_bag_(Sz, K, Res, Ref, I) -> 1296 | case Res of 1297 | {ok, K} -> 1298 | do_delete_bag_(Sz, K, ?leveldb:iterator_move(I, next), 1299 | Ref, I); 1300 | {ok, <> = Key} -> 1301 | [Key | 1302 | do_delete_bag_(Sz, K, ?leveldb:iterator_move(I, next), 1303 | Ref, I)]; 1304 | _ -> 1305 | [] 1306 | end. 1307 | 1308 | do_match_delete(Pat, #st{ets = Ets, ref = Ref, tab = Tab, type = Type, 1309 | record_name = RecName, 1310 | maintain_size = MaintainSize}) -> 1311 | Fun = fun(_, Key, Acc) -> [Key|Acc] end, 1312 | Keys = do_fold(Ref, Tab, Type, Fun, [], [{Pat,[],['$_']}], 30, RecName), 1313 | case {Keys, MaintainSize} of 1314 | {[], _} -> 1315 | ok; 1316 | {_, false} -> 1317 | ?leveldb:write(Ref, [{delete, K} || K <- Keys], []), 1318 | ok; 1319 | {_, true} -> 1320 | CurSz = read_info(size, 0, Ets), 1321 | NewSz = max(CurSz - length(Keys), 0), 1322 | {Ki, Vi} = info_obj(size, NewSz), 1323 | ?leveldb:write(Ref, [{put, Ki, Vi} | 1324 | [{delete, K} || K <- Keys]], []), 1325 | ets_insert_info(Ets, size, NewSz), 1326 | ok 1327 | end. 1328 | 1329 | recover_size_info(#st{ ref = Ref 1330 | , tab = Tab 1331 | , type = Type 1332 | , record_name = RecName 1333 | , maintain_size = MaintainSize 1334 | } = St) -> 1335 | %% TODO: shall_update_size_info is obsolete, remove 1336 | case shall_update_size_info(Tab) of 1337 | true -> 1338 | Sz = do_fold(Ref, Tab, Type, fun(_, Acc) -> Acc+1 end, 1339 | 0, [{'_',[],['$_']}], 3, RecName), 1340 | write_info_(size, Sz, St); 1341 | false -> 1342 | case MaintainSize of 1343 | true -> 1344 | %% info initialized by leveldb_to_ets/2 1345 | %% TODO: if there is no stored size, recompute it 1346 | ignore; 1347 | false -> 1348 | %% size is not maintained, ensure it's marked accordingly 1349 | delete_info_(size, St) 1350 | end 1351 | end, 1352 | St. 1353 | 1354 | shall_update_size_info({_, index, _}) -> 1355 | false; 1356 | shall_update_size_info(Tab) -> 1357 | property(Tab, update_size_info, false). 1358 | 1359 | should_maintain_size(Tab) -> 1360 | property(Tab, maintain_size, false). 1361 | 1362 | property(Tab, Prop, Default) -> 1363 | try mnesia:read_table_property(Tab, Prop) of 1364 | {Prop, P} -> 1365 | P 1366 | catch 1367 | error:_ -> Default; 1368 | exit:_ -> Default 1369 | end. 1370 | 1371 | write_info_(Item, Val, #st{ets = Ets, ref = Ref}) -> 1372 | leveldb_insert_info(Ref, Item, Val), 1373 | ets_insert_info(Ets, Item, Val). 1374 | 1375 | ets_insert_info(Ets, Item, Val) -> 1376 | ets:insert(Ets, {{info, Item}, Val}). 1377 | 1378 | ets_delete_info(Ets, Item) -> 1379 | ets:delete(Ets, {info, Item}). 1380 | 1381 | leveldb_insert_info(Ref, Item, Val) -> 1382 | EncKey = info_key(Item), 1383 | EncVal = encode_val(Val), 1384 | ?leveldb:put(Ref, EncKey, EncVal, []). 1385 | 1386 | leveldb_delete_info(Ref, Item) -> 1387 | EncKey = info_key(Item), 1388 | ?leveldb:delete(Ref, EncKey, []). 1389 | 1390 | info_obj(Item, Val) -> 1391 | {info_key(Item), encode_val(Val)}. 1392 | 1393 | info_key(Item) -> 1394 | <>. 1395 | 1396 | delete_info_(Item, #st{ets = Ets, ref = Ref}) -> 1397 | leveldb_delete_info(Ref, Item), 1398 | ets_delete_info(Ets, Item). 1399 | 1400 | read_info(Item, Default, Ets) -> 1401 | case ets:lookup(Ets, {info,Item}) of 1402 | [] -> 1403 | Default; 1404 | [{_,Val}] -> 1405 | Val 1406 | end. 1407 | 1408 | tab_name(icache, Tab) -> 1409 | list_to_atom("mnesia_ext_icache_" ++ tabname(Tab)). 1410 | 1411 | proc_name(_Alias, Tab) -> 1412 | list_to_atom("mnesia_ext_proc_" ++ tabname(Tab)). 1413 | 1414 | 1415 | %% ---------------------------------------------------------------------------- 1416 | %% PRIVATE SELECT MACHINERY 1417 | %% ---------------------------------------------------------------------------- 1418 | 1419 | do_select(Ref, Tab, Type, MS, Limit, RecName) -> 1420 | do_select(Ref, Tab, Type, MS, false, Limit, RecName). 1421 | 1422 | do_select(Ref, Tab, _Type, MS, AccKeys, Limit, RecName) when is_boolean(AccKeys) -> 1423 | Keypat = keypat(MS, keypos(Tab)), 1424 | Sel = #sel{tab = Tab, 1425 | ref = Ref, 1426 | keypat = Keypat, 1427 | compiled_ms = ets:match_spec_compile(MS), 1428 | record_name = RecName, 1429 | limit = Limit}, 1430 | with_iterator(Ref, fun(I) -> i_do_select(I, Sel, AccKeys, []) end). 1431 | 1432 | i_do_select(I, #sel{keypat = Pfx, 1433 | compiled_ms = MS, 1434 | limit = Limit} = Sel, AccKeys, Acc) -> 1435 | StartKey = 1436 | case Pfx of 1437 | <<>> -> 1438 | <>; 1439 | _ -> 1440 | Pfx 1441 | end, 1442 | select_traverse(?leveldb:iterator_move(I, StartKey), Limit, 1443 | Pfx, MS, I, Sel, AccKeys, Acc). 1444 | 1445 | select_traverse({ok, K, V}, Limit, Pfx, MS, I, #sel{tab = Tab, record_name = RecName} = Sel, 1446 | AccKeys, Acc) -> 1447 | case is_prefix(Pfx, K) of 1448 | true -> 1449 | Rec = set_record(keypos(Tab), decode_val(V), decode_key(K), RecName), 1450 | case ets:match_spec_run([Rec], MS) of 1451 | [] -> 1452 | select_traverse( 1453 | ?leveldb:iterator_move(I, next), Limit, Pfx, MS, 1454 | I, Sel, AccKeys, Acc); 1455 | [Match] -> 1456 | Acc1 = if AccKeys -> 1457 | [{K, Match}|Acc]; 1458 | true -> 1459 | [Match|Acc] 1460 | end, 1461 | traverse_continue(K, decr(Limit), Pfx, MS, I, Sel, AccKeys, Acc1) 1462 | end; 1463 | false -> 1464 | {lists:reverse(Acc), '$end_of_table'} 1465 | end; 1466 | select_traverse({error, _}, _, _, _, _, _, _, Acc) -> 1467 | {lists:reverse(Acc), '$end_of_table'}. 1468 | 1469 | is_prefix(A, B) when is_binary(A), is_binary(B) -> 1470 | Sa = byte_size(A), 1471 | case B of 1472 | <> -> 1473 | true; 1474 | _ -> 1475 | false 1476 | end. 1477 | 1478 | decr(I) when is_integer(I) -> 1479 | I-1; 1480 | decr(infinity) -> 1481 | infinity. 1482 | 1483 | traverse_continue(K, 0, Pfx, MS, _I, #sel{limit = Limit, ref = Ref} = Sel, AccKeys, Acc) -> 1484 | {lists:reverse(Acc), 1485 | fun() -> 1486 | with_iterator(Ref, 1487 | fun(NewI) -> 1488 | select_traverse(iterator_next(NewI, K), 1489 | Limit, Pfx, MS, NewI, Sel, 1490 | AccKeys, []) 1491 | end) 1492 | end}; 1493 | traverse_continue(_K, Limit, Pfx, MS, I, Sel, AccKeys, Acc) -> 1494 | select_traverse(?leveldb:iterator_move(I, next), Limit, Pfx, MS, I, Sel, AccKeys, Acc). 1495 | 1496 | iterator_next(I, K) -> 1497 | case ?leveldb:iterator_move(I, K) of 1498 | {ok, K, _} -> 1499 | ?leveldb:iterator_move(I, next); 1500 | Other -> 1501 | Other 1502 | end. 1503 | 1504 | keypat([H|T], KeyPos) -> 1505 | keypat(T, KeyPos, keypat_pfx(H, KeyPos)). 1506 | 1507 | keypat(_, _, <<>>) -> <<>>; 1508 | keypat([H|T], KeyPos, Pfx0) -> 1509 | Pfx = keypat_pfx(H, KeyPos), 1510 | keypat(T, KeyPos, common_prefix(Pfx, Pfx0)); 1511 | keypat([], _, Pfx) -> 1512 | Pfx. 1513 | 1514 | common_prefix(<>, <>) -> 1515 | <>; 1516 | common_prefix(_, _) -> 1517 | <<>>. 1518 | 1519 | keypat_pfx({HeadPat,_Gs,_}, KeyPos) when is_tuple(HeadPat) -> 1520 | KP = element(KeyPos, HeadPat), 1521 | mnesia_eleveldb_sext:prefix(KP); 1522 | keypat_pfx(_, _) -> 1523 | <<>>. 1524 | 1525 | %% ---------------------------------------------------------------------------- 1526 | %% COMMON PRIVATE 1527 | %% ---------------------------------------------------------------------------- 1528 | 1529 | %% Note that since a callback can be used as an indexing backend, we 1530 | %% cannot assume that keypos will always be 2. For indexes, the tab 1531 | %% name will be {Tab, index, Pos}, and The object structure will be 1532 | %% {{IxKey,Key}} for an ordered_set index, and {IxKey,Key} for a bag 1533 | %% index. 1534 | %% 1535 | keypos({_, index, _}) -> 1536 | 1; 1537 | keypos({_, retainer, _}) -> 1538 | 2; 1539 | keypos(Tab) when is_atom(Tab) -> 1540 | 2. 1541 | 1542 | encode_key(Key) -> 1543 | mnesia_eleveldb_sext:encode(Key). 1544 | 1545 | decode_key(CodedKey) -> 1546 | case mnesia_eleveldb_sext:partial_decode(CodedKey) of 1547 | {full, Result, _} -> 1548 | Result; 1549 | _ -> 1550 | error(badarg, CodedKey) 1551 | end. 1552 | 1553 | encode_val(Val) -> 1554 | term_to_binary(Val). 1555 | 1556 | decode_val(CodedVal) -> 1557 | binary_to_term(CodedVal). 1558 | 1559 | %% Update key and record name in the record with new values. 1560 | %% Used both in write (set both to []) and read paths (replace 1561 | %% with decoded key and known record name, respectively). 1562 | %% If key position is 1 then don't set the record name, 1563 | %% only the key -- see keypos/1. 1564 | set_record(1, Record, KeyVal, _RecName) -> 1565 | setelement(1, Record, KeyVal); 1566 | set_record(KeyPos, Record, KeyVal, RecName) -> 1567 | setelement(1, setelement(KeyPos, Record, KeyVal), RecName). 1568 | 1569 | create_mountpoint(Tab) -> 1570 | MPd = data_mountpoint(Tab), 1571 | case filelib:is_dir(MPd) of 1572 | false -> 1573 | file:make_dir(MPd), 1574 | ok; 1575 | true -> 1576 | Dir = mnesia_lib:dir(), 1577 | case lists:prefix(Dir, MPd) of 1578 | true -> 1579 | ok; 1580 | false -> 1581 | {error, exists} 1582 | end 1583 | end. 1584 | 1585 | %% delete_mountpoint(Tab) -> 1586 | %% MPd = data_mountpoint(Tab), 1587 | %% assert_proper_mountpoint(Tab, MPd), 1588 | %% ok = destroy_db(MPd, []). 1589 | 1590 | assert_proper_mountpoint(_Tab, _MPd) -> 1591 | %% TODO: not yet implemented. How to verify that the MPd var points 1592 | %% to the directory we actually want deleted? 1593 | ok. 1594 | 1595 | data_mountpoint(Tab) -> 1596 | Dir = mnesia_monitor:get_env(dir), 1597 | filename:join(Dir, tabname(Tab) ++ ".extldb"). 1598 | 1599 | tabname({Tab, index, {{Pos},_}}) -> 1600 | atom_to_list(Tab) ++ "-=" ++ atom_to_list(Pos) ++ "=-_ix"; 1601 | tabname({Tab, index, {Pos,_}}) -> 1602 | atom_to_list(Tab) ++ "-" ++ integer_to_list(Pos) ++ "-_ix"; 1603 | tabname({Tab, retainer, Name}) -> 1604 | atom_to_list(Tab) ++ "-" ++ retainername(Name) ++ "-_RET"; 1605 | tabname(Tab) when is_atom(Tab) -> 1606 | atom_to_list(Tab) ++ "-_tab". 1607 | 1608 | retainername(Name) when is_atom(Name) -> 1609 | atom_to_list(Name); 1610 | retainername(Name) when is_list(Name) -> 1611 | try binary_to_list(list_to_binary(Name)) 1612 | catch 1613 | error:_ -> 1614 | lists:flatten(io_lib:write(Name)) 1615 | end; 1616 | retainername(Name) -> 1617 | lists:flatten(io_lib:write(Name)). 1618 | 1619 | related_resources(Tab) -> 1620 | TabS = atom_to_list(Tab), 1621 | Dir = mnesia_monitor:get_env(dir), 1622 | case file:list_dir(Dir) of 1623 | {ok, Files} -> 1624 | lists:flatmap( 1625 | fun(F) -> 1626 | Full = filename:join(Dir, F), 1627 | case is_index_dir(F, TabS) of 1628 | false -> 1629 | case is_retainer_dir(F, TabS) of 1630 | false -> 1631 | []; 1632 | {true, Name} -> 1633 | [{{Tab, retainer, Name}, Full}] 1634 | end; 1635 | {true, Pos} -> 1636 | [{{Tab, index, {Pos,ordered}}, Full}] 1637 | end 1638 | end, Files); 1639 | _ -> 1640 | [] 1641 | end. 1642 | 1643 | is_index_dir(F, TabS) -> 1644 | case re:run(F, TabS ++ "-([0-9]+)-_ix.extldb", [{capture, [1], list}]) of 1645 | nomatch -> 1646 | false; 1647 | {match, [P]} -> 1648 | {true, list_to_integer(P)} 1649 | end. 1650 | 1651 | is_retainer_dir(F, TabS) -> 1652 | case re:run(F, TabS ++ "-(.+)-_RET", [{capture, [1], list}]) of 1653 | nomatch -> 1654 | false; 1655 | {match, [Name]} -> 1656 | {true, Name} 1657 | end. 1658 | 1659 | get_ref(Alias, Tab) -> 1660 | call(Alias, Tab, get_ref). 1661 | 1662 | fold(Alias, Tab, Fun, Acc, MS, N) -> 1663 | {Ref, Type, RecName} = get_ref(Alias, Tab), 1664 | do_fold(Ref, Tab, Type, Fun, Acc, MS, N, RecName). 1665 | 1666 | %% can be run on the server side. 1667 | do_fold(Ref, Tab, Type, Fun, Acc, MS, N, RecName) -> 1668 | {AccKeys, F} = 1669 | if is_function(Fun, 3) -> 1670 | {true, fun({K,Obj}, Acc1) -> 1671 | Fun(Obj, K, Acc1) 1672 | end}; 1673 | is_function(Fun, 2) -> 1674 | {false, Fun} 1675 | end, 1676 | do_fold1(do_select(Ref, Tab, Type, MS, AccKeys, N, RecName), F, Acc). 1677 | 1678 | do_fold1('$end_of_table', _, Acc) -> 1679 | Acc; 1680 | do_fold1({L, Cont}, Fun, Acc) -> 1681 | Acc1 = lists:foldl(Fun, Acc, L), 1682 | do_fold1(select(Cont), Fun, Acc1). 1683 | 1684 | is_wild('_') -> 1685 | true; 1686 | is_wild(A) when is_atom(A) -> 1687 | case atom_to_list(A) of 1688 | "\$" ++ S -> 1689 | try begin 1690 | _ = list_to_integer(S), 1691 | true 1692 | end 1693 | catch 1694 | error:_ -> 1695 | false 1696 | end; 1697 | _ -> 1698 | false 1699 | end; 1700 | is_wild(_) -> 1701 | false. 1702 | -------------------------------------------------------------------------------- /src/mnesia_eleveldb_app.erl: -------------------------------------------------------------------------------- 1 | %%---------------------------------------------------------------- 2 | %% Copyright (c) 2013-2016 Klarna AB 3 | %% 4 | %% This file is provided to you under the Apache License, 5 | %% Version 2.0 (the "License"); you may not use this file 6 | %% except in compliance with the License. You may obtain 7 | %% a copy of the License at 8 | %% 9 | %% http://www.apache.org/licenses/LICENSE-2.0 10 | %% 11 | %% Unless required by applicable law or agreed to in writing, 12 | %% software distributed under the License is distributed on an 13 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | %% KIND, either express or implied. See the License for the 15 | %% specific language governing permissions and limitations 16 | %% under the License. 17 | %%---------------------------------------------------------------- 18 | 19 | -module(mnesia_eleveldb_app). 20 | 21 | -behaviour(application). 22 | 23 | %% Application callbacks 24 | -export([start/2, stop/1]). 25 | 26 | %% =================================================================== 27 | %% Application callbacks 28 | %% =================================================================== 29 | 30 | start(_StartType, _StartArgs) -> 31 | mnesia_eleveldb_sup:start_link(). 32 | 33 | stop(_State) -> 34 | ok. 35 | -------------------------------------------------------------------------------- /src/mnesia_eleveldb_params.erl: -------------------------------------------------------------------------------- 1 | %%---------------------------------------------------------------- 2 | %% Copyright (c) 2013-2016 Klarna AB 3 | %% 4 | %% This file is provided to you under the Apache License, 5 | %% Version 2.0 (the "License"); you may not use this file 6 | %% except in compliance with the License. You may obtain 7 | %% a copy of the License at 8 | %% 9 | %% http://www.apache.org/licenses/LICENSE-2.0 10 | %% 11 | %% Unless required by applicable law or agreed to in writing, 12 | %% software distributed under the License is distributed on an 13 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | %% KIND, either express or implied. See the License for the 15 | %% specific language governing permissions and limitations 16 | %% under the License. 17 | %%---------------------------------------------------------------- 18 | 19 | -module(mnesia_eleveldb_params). 20 | 21 | -behaviour(gen_server). 22 | 23 | -export([lookup/2, 24 | store/2, 25 | delete/1]). 26 | 27 | -export([start_link/0, 28 | init/1, 29 | handle_call/3, 30 | handle_cast/2, 31 | handle_info/2, 32 | terminate/2, 33 | code_change/3]). 34 | 35 | -include("mnesia_eleveldb_tuning.hrl"). 36 | 37 | -define(KB, 1024). 38 | -define(MB, 1024 * 1024). 39 | -define(GB, 1024 * 1024 * 1024). 40 | 41 | -ifdef(DEBUG). 42 | -define(dbg(Fmt, Args), io:fwrite(user,"~p:~p: "++(Fmt),[?MODULE,?LINE|Args])). 43 | -else. 44 | -define(dbg(Fmt, Args), ok). 45 | -endif. 46 | 47 | lookup(Tab, Default) -> 48 | try ets:lookup(?MODULE, Tab) of 49 | [{_, Params}] -> 50 | Params; 51 | [] -> 52 | Default 53 | catch error:badarg -> 54 | Default 55 | end. 56 | 57 | store(Tab, Params) -> 58 | ets:insert(?MODULE, {Tab, Params}). 59 | 60 | delete(Tab) -> 61 | ets:delete(?MODULE, Tab). 62 | 63 | start_link() -> 64 | case ets:info(?MODULE, name) of 65 | undefined -> 66 | ets:new(?MODULE, [ordered_set, public, named_table]), 67 | load_tuning_parameters(); 68 | _ -> 69 | ok 70 | end, 71 | gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). 72 | 73 | init(_) -> 74 | {ok, []}. 75 | 76 | handle_call(_, _, S) -> {reply, error, S}. 77 | handle_cast(_, S) -> {noreply, S}. 78 | handle_info(_, S) -> {noreply, S}. 79 | terminate(_, _) -> ok. 80 | code_change(_, S, _) -> {ok, S}. 81 | 82 | load_tuning_parameters() -> 83 | case application:get_env(mnesia_eleveldb, tuning_params) of 84 | {ok, Ps} -> 85 | case Ps of 86 | {consult, F} -> consult(F); 87 | {script, F} -> script(F); 88 | _ when is_list(Ps) -> 89 | store_params(Ps) 90 | end; 91 | _ -> 92 | ok 93 | end. 94 | 95 | consult(F) -> 96 | case file:consult(F) of 97 | {ok, Terms} -> 98 | store_params(Terms); 99 | {error, Reason} -> 100 | {error, {Reason, F}} 101 | end. 102 | 103 | script(F) -> 104 | case file:script(F) of 105 | {ok, Terms} -> 106 | store_params(Terms); 107 | {error, Reason} -> 108 | {error, {Reason, F}} 109 | end. 110 | 111 | store_params(Params) -> 112 | _ = lists:foreach(fun({_,S}) -> valid_size(S) end, Params), 113 | NTabs = length(Params), 114 | Env0= mnesia_eleveldb_tuning:describe_env(), 115 | Env = Env0#tuning{n_tabs = NTabs}, 116 | ?dbg("Env = ~p~n", [Env]), 117 | TotalFiles = lists:sum([mnesia_eleveldb_tuning:max_files(Sz) || 118 | {_, Sz} <- Params]), 119 | ?dbg("TotalFiles = ~p~n", [TotalFiles]), 120 | MaxFs = Env#tuning.max_files, 121 | ?dbg("MaxFs = ~p~n", [MaxFs]), 122 | FsHeadroom = MaxFs * 0.6, 123 | ?dbg("FsHeadroom = ~p~n", [FsHeadroom]), 124 | FilesFactor = if TotalFiles =< FsHeadroom -> 125 | 1; % don't have to scale down 126 | true -> 127 | FsHeadroom / TotalFiles 128 | end, 129 | Env1 = Env#tuning{files_factor = FilesFactor}, 130 | ?dbg("Env1 = ~p~n", [Env1]), 131 | lists:foreach( 132 | fun({Tab, Sz}) when is_atom(Tab); 133 | is_atom(element(1,Tab)), 134 | is_integer(element(2,Tab)) -> 135 | ets:insert(?MODULE, {Tab, ldb_params(Sz, Env1, Tab)}) 136 | end, Params). 137 | 138 | ldb_params(Sz, Env, _Tab) -> 139 | MaxFiles = mnesia_eleveldb_tuning:max_files(Sz) * Env#tuning.files_factor, 140 | Opts = if Env#tuning.avail_ram > 100 -> % Gigabytes 141 | [{write_buffer_size, mnesia_eleveldb_tuning:write_buffer(Sz)}, 142 | {cache_size, mnesia_eleveldb_tuning:cache(Sz)}]; 143 | true -> 144 | [] 145 | end, 146 | [{max_open_files, MaxFiles} | Opts]. 147 | 148 | valid_size({I,U}) when is_number(I) -> 149 | true = lists:member(U, [k,m,g]). 150 | -------------------------------------------------------------------------------- /src/mnesia_eleveldb_sext.erl: -------------------------------------------------------------------------------- 1 | %% -*- erlang-indent-level: 4; indent-tabs-mode: nil 2 | %%============================================================================== 3 | %% Copyright 2014-2016 Ulf Wiger 4 | %% 5 | %% Licensed under the Apache License, Version 2.0 (the "License"); 6 | %% you may not use this file except in compliance with the License. 7 | %% You may obtain a copy of the License at 8 | %% 9 | %% http://www.apache.org/licenses/LICENSE-2.0 10 | %% 11 | %% Unless required by applicable law or agreed to in writing, software 12 | %% distributed under the License is distributed on an "AS IS" BASIS, 13 | %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | %% See the License for the specific language governing permissions and 15 | %% limitations under the License. 16 | %%============================================================================== 17 | 18 | %% @author Ulf Wiger 19 | %% @doc Sortable serialization library 20 | %% @end 21 | 22 | -module(mnesia_eleveldb_sext). 23 | 24 | -export([encode/1, encode/2, decode/1, decode_next/1]). 25 | -export([encode_hex/1, decode_hex/1]). 26 | -export([encode_sb32/1, decode_sb32/1]). 27 | -export([prefix/1, 28 | partial_decode/1]). 29 | -export([prefix_hex/1]). 30 | -export([prefix_sb32/1]). 31 | -export([to_sb32/1, from_sb32/1]). 32 | -export([to_hex/1, from_hex/1]). 33 | 34 | 35 | -define(negbig , 8). 36 | -define(neg4 , 9). 37 | -define(pos4 , 10). 38 | -define(posbig , 11). 39 | -define(atom , 12). 40 | -define(reference, 13). 41 | -define(port , 14). 42 | -define(pid , 15). 43 | -define(tuple , 16). 44 | -define(list , 17). 45 | -define(binary , 18). 46 | -define(bin_tail , 19). 47 | 48 | -define(is_sext(X), 49 | X==?negbig; 50 | X==?neg4; 51 | X==?pos4; 52 | X==?posbig; 53 | X==?atom; 54 | X==?reference; 55 | X==?port; 56 | X==?pid; 57 | X==?tuple; 58 | X==?list; 59 | X==?binary; 60 | X==?bin_tail). 61 | 62 | -define(IMAX1, 16#ffffFFFFffffFFFF). 63 | 64 | -ifdef(DEBUG). 65 | -define(dbg(Fmt, Args), 66 | case os:getenv("MNESIA_SEXT_DEBUG") of 67 | false -> ok; 68 | _ -> io:fwrite(user,"~p:~p: "++(Fmt),[?MODULE,?LINE|Args]) 69 | end). 70 | -else. 71 | -define(dbg(F,A), ok). 72 | -endif. 73 | 74 | %% @spec encode(T::term()) -> binary() 75 | %% @doc Encodes any Erlang term into a binary. 76 | %% The lexical sorting properties of the encoded binary match those of the 77 | %% original Erlang term. That is, encoded terms sort the same way as the 78 | %% original terms would. 79 | %% @end 80 | %% 81 | encode(X) -> encode(X, false). 82 | 83 | %% @spec encode(T::term(), Legacy::boolean()) -> binary() 84 | %% @doc Encodes an Erlang term using legacy bignum encoding. 85 | %% On March 4 2013, Basho noticed that encoded bignums didn't always sort 86 | %% properly. This bug has been fixed, but the encoding of bignums necessarily 87 | %% changed in an incompatible way. 88 | %% 89 | %% The new decode/1 version can read the old bignum format, but the old 90 | %% version obviously cannot read the new. Using `encode(Term, true)', the term 91 | %% will be encoded using the old format. 92 | %% 93 | %% Use only as transition support. This function will be deprecated in time. 94 | %% @end 95 | encode(X, Legacy) when is_tuple(X) -> encode_tuple(X, Legacy); 96 | encode(X, Legacy) when is_list(X) -> encode_list(X, Legacy); 97 | encode(X, _) when is_pid(X) -> encode_pid(X); 98 | encode(X, _) when is_port(X) -> encode_port(X); 99 | encode(X, _) when is_reference(X) -> encode_ref(X); 100 | encode(X, Legacy) when is_number(X) -> encode_number(X, Legacy); 101 | encode(X, _) when is_binary(X) -> encode_binary(X); 102 | encode(X, _) when is_bitstring(X) -> encode_bitstring(X); 103 | encode(X, _) when is_atom(X) -> encode_atom(X). 104 | 105 | %% @spec encode_sb32(Term::any()) -> binary() 106 | %% @doc Encodes any Erlang term into an sb32-encoded binary. 107 | %% This is similar to {@link encode/1}, but produces an octet string that 108 | %% can be used without escaping in file names (containing only the characters 109 | %% 0..9, A..V and '-'). The sorting properties are preserved. 110 | %% 111 | %% Note: The encoding used is inspired by the base32 encoding described in 112 | %% RFC3548, but uses a different alphabet in order to preserve the sort order. 113 | %% @end 114 | %% 115 | encode_sb32(Term) -> 116 | to_sb32(encode(Term)). 117 | 118 | %% @spec encode_hex(Term::any()) -> binary() 119 | %% @doc Encodes any Erlang term into a hex-encoded binary. 120 | %% This is similar to {@link encode/1}, but produces an octet string that 121 | %% can be used without escaping in file names (containing only the characters 122 | %% 0..9 and A..F). The sorting properties are preserved. 123 | %% 124 | %% Note: The encoding used is regular hex-encoding, with the proviso that only 125 | %% capital letters are used (mixing upper- and lowercase characters would break 126 | %% the sorting property). 127 | %% @end 128 | %% 129 | encode_hex(Term) -> 130 | to_hex(encode(Term)). 131 | 132 | %% @spec prefix(X::term()) -> binary() 133 | %% @doc Encodes a binary for prefix matching of similar encoded terms. 134 | %% Lists and tuples can be prefixed by using the '_' marker, 135 | %% similarly to Erlang match specifications. For example: 136 | %%
    137 | %%
  • prefix({1,2,'_','_'}) will result in a binary that is 138 | %% the same as the first part of any encoded 4-tuple with the first two 139 | %% elements being 1 and 2. The prefix algorithm will search for the 140 | %% first '_', and treat all following elements as if they 141 | %% were '_'.
    • 142 | %%
  • prefix([1,2|'_']) will result in a binary that is the 143 | %% same as the first part of any encoded list where the first two elements 144 | %% are 1 and 2. prefix([1,2,'_']) will give the same result, 145 | %% as the prefix pattern is the same for all lists starting with 146 | %% `[1,2|...]'.
    • 147 | %%
  • `prefix(Binary)' will result in a binary that is the same as the 148 | %% encoded version of Binary, except that, instead of padding and 149 | %% terminating, the encoded binary is truncated to the longest byte-aligned 150 | %% binary. The same is done for bitstrings.
    • 151 | %%
  • prefix({1,[1,2|'_'],'_'}) will prefix-encode the second 152 | %% element, and let it end the resulting binary. This prefix will match 153 | %% any 3-tuple where the first element is 1 and the second element is a 154 | %% list where the first two elements are 1 and 2.
    • 155 | %%
  • prefix([1,[1|'_']|'_']) will result in a prefix that 156 | %% matches all lists where the first element is 1 and the second element is 157 | %% a list where the first element is 1.
    • 158 | %%
  • For all other data types, the prefix is the same as the encoded term. 159 | %%
    • 160 | %%
161 | %% @end 162 | %% 163 | prefix(X) -> 164 | {_, P} = enc_prefix(X), 165 | P. 166 | 167 | enc_prefix(X) when is_tuple(X) -> prefix_tuple(X); 168 | enc_prefix(X) when is_list(X) -> prefix_list(X); 169 | enc_prefix(X) when is_pid(X) -> {false, encode_pid(X)}; 170 | enc_prefix(X) when is_port(X) -> {false, encode_port(X)}; 171 | enc_prefix(X) when is_reference(X) -> {false, encode_ref(X)}; 172 | enc_prefix(X) when is_number(X) -> {false, encode_number(X)}; 173 | enc_prefix(X) when is_binary(X) -> prefix_binary(X); 174 | enc_prefix(X) when is_bitstring(X) -> prefix_bitstring(X); 175 | enc_prefix(X) when is_atom(X) -> 176 | case is_wild(X) of 177 | true -> 178 | {true, <<>>}; 179 | false -> 180 | {false, encode_atom(X)} 181 | end. 182 | 183 | %% @spec prefix_sb32(X::term()) -> binary() 184 | %% @doc Generates an sb32-encoded binary for prefix matching. 185 | %% This is similar to {@link prefix/1}, but generates a prefix for binaries 186 | %% encoded with {@link encode_sb32/1}, rather than {@link encode/1}. 187 | %% @end 188 | %% 189 | prefix_sb32(X) -> 190 | chop_prefix_tail(to_sb32(prefix(X))). 191 | 192 | %% @spec prefix_hex(X::term()) -> binary() 193 | %% @doc Generates a hex-encoded binary for prefix matching. 194 | %% This is similar to {@link prefix/1}, but generates a prefix for binaries 195 | %% encoded with {@link encode_hex/1}, rather than {@link encode/1}. 196 | %% @end 197 | %% 198 | prefix_hex(X) -> 199 | to_hex(prefix(X)). 200 | 201 | %% Must chop of the pad character and the last encoded unit (which, if pad 202 | %% characters are present, is not a whole byte) 203 | %% 204 | chop_prefix_tail(Bin) -> 205 | Sz = byte_size(Bin), 206 | Sz6 = Sz-7, Sz4 = Sz - 5, Sz3 = Sz - 4, Sz1 = Sz - 2, 207 | case Bin of 208 | << P:Sz6/binary, _, "------" >> -> P; 209 | << P:Sz4/binary, _, "----" >> -> P; 210 | << P:Sz3/binary, _, "---" >> -> P; 211 | << P:Sz1/binary, _, "-" >> -> P; 212 | _ -> Bin 213 | end. 214 | 215 | %% @spec decode(B::binary()) -> term() 216 | %% @doc Decodes a binary generated using the function {@link sext:encode/1}. 217 | %% @end 218 | %% 219 | decode(Elems) -> 220 | case decode_next(Elems) of 221 | {Term, <<>>} -> Term; 222 | Other -> erlang:error(badarg, Other) 223 | end. 224 | 225 | %% spec decode_sb32(B::binary()) -> term() 226 | %% @doc Decodes a binary generated using the function {@link encode_sb32/1}. 227 | %% @end 228 | %% 229 | decode_sb32(Data) -> 230 | decode(from_sb32(Data)). 231 | 232 | decode_hex(Data) -> 233 | decode(from_hex(Data)). 234 | 235 | encode_tuple(T, Legacy) -> 236 | Sz = size(T), 237 | encode_tuple_elems(1, Sz, T, <>, Legacy). 238 | 239 | prefix_tuple(T) -> 240 | Sz = size(T), 241 | Elems = tuple_to_list(T), 242 | prefix_tuple_elems(Elems, <>). 243 | 244 | %% It's easier to iterate over a tuple by converting it to a list, but 245 | %% since the tuple /can/ be huge, let's do it this way. 246 | encode_tuple_elems(P, Sz, T, Acc, Legacy) when P =< Sz -> 247 | E = encode(element(P,T), Legacy), 248 | encode_tuple_elems(P+1, Sz, T, <>, Legacy); 249 | encode_tuple_elems(_, _, _, Acc, _) -> 250 | Acc. 251 | 252 | prefix_tuple_elems([A|T], Acc) when is_atom(A) -> 253 | case is_wild(A) of 254 | true -> 255 | {true, Acc}; 256 | false -> 257 | E = encode(A), 258 | prefix_tuple_elems(T, <>) 259 | end; 260 | prefix_tuple_elems([H|T], Acc) -> 261 | case enc_prefix(H) of 262 | {true, P} -> 263 | {true, <>}; 264 | {false, E} -> 265 | prefix_tuple_elems(T, <>) 266 | end; 267 | prefix_tuple_elems([], Acc) -> 268 | {false, Acc}. 269 | 270 | encode_list(L, Legacy) -> 271 | encode_list_elems(L, <>, Legacy). 272 | 273 | prefix_list(L) -> 274 | prefix_list_elems(L, <>). 275 | 276 | encode_binary(B) -> 277 | Enc = encode_bin_elems(B), 278 | <>. 279 | 280 | prefix_binary(B) -> 281 | Enc = encode_bin_elems(B), 282 | {false, <>}. 283 | 284 | encode_bitstring(B) -> 285 | Enc = encode_bits_elems(B), 286 | <>. 287 | 288 | prefix_bitstring(B) -> 289 | Enc = encode_bits_elems(B), 290 | {false, <>}. 291 | 292 | encode_pid(P) -> 293 | case term_to_binary(P) of 294 | <<131,88,119,ALen:8,Name:ALen/binary,NS:8/binary,C:32>> -> 295 | encode_pid_new(Name, NS, C); 296 | <<131,88,100,ALen:16,Name:ALen/binary,NS:8/binary,C:32>> -> 297 | encode_pid_new(Name, NS, C); 298 | <<131,103,100,ALen:16,Name:ALen/binary,NS:8/binary,C:8>> -> 299 | true = C =< 3, 300 | encode_pid(Name, NS, <>) 301 | end. 302 | 303 | encode_pid_new(Name, NS, C) -> 304 | CBin = 305 | case C > 3 of 306 | true -> <<255, C:32>>; 307 | false -> <> 308 | end, 309 | encode_pid(Name, NS, CBin). 310 | 311 | encode_pid(Name, NS, C) -> 312 | NameEnc = encode_bin_elems(Name), 313 | <>. 314 | 315 | encode_port(P) -> 316 | case term_to_binary(P) of 317 | <<131,120,119,ALen:8,Name:ALen/binary,N:64,C:32>> -> 318 | case N bsr 28 of 319 | 0 -> encode_port_new(Name, <>, C); 320 | _ -> 321 | %% N was limited to 28 bits previously, meaning the initial byte 322 | %% in its binary was =< 15. We therefore prefix the 8-byte N with 323 | %% a byte with value 16 to signal the V4 format, and to ensure V4 324 | %% formats sort consistently with the previous format. In this 325 | %% case we don't need to try shortening the C(reation) field. 326 | encode_port(Name, <<16,N:64>>, <>) 327 | end; 328 | <<131,89,100,ALen:16,Name:ALen/binary,N:32,C:32>> -> 329 | 0 = N bsr 28, % assert 330 | encode_port_new(Name, <>, C); 331 | <<131,102,100,ALen:16,Name:ALen/binary,N:32,C:8>> -> 332 | 0 = N bsr 28, % assert 333 | true = C =< 3, 334 | encode_port(Name, <>, <>) 335 | end. 336 | 337 | encode_port_new(Name, N, C) -> 338 | CBin = 339 | case C > 3 of 340 | true -> <<255, C:32>>; 341 | false -> <> 342 | end, 343 | encode_port(Name, N, CBin). 344 | 345 | encode_port(Name, N, C) -> 346 | NameEnc = encode_bin_elems(Name), 347 | <>. 348 | 349 | encode_ref(R) -> 350 | case term_to_binary(R) of 351 | <<131,90,_Len:16,119,NLen:8,Name:NLen/binary,C:32,Rest/binary>> -> 352 | encode_ref_newer(Name, C, Rest); 353 | <<131,90,_Len:16,100,NLen:16,Name:NLen/binary,C:32,Rest/binary>> -> 354 | encode_ref_newer(Name, C, Rest); 355 | <<131,114,_Len:16,100,NLen:16,Name:NLen/binary,C:8,Rest/binary>> -> 356 | true = C =< 3, 357 | encode_ref(Name, <>) 358 | end. 359 | 360 | encode_ref_newer(Name, C, Rest) -> 361 | NewRest = 362 | case C > 3 of 363 | true -> <<255, C:32, Rest/binary>>; 364 | false -> <> 365 | end, 366 | encode_ref(Name, NewRest). 367 | 368 | encode_ref(Name, Rest) -> 369 | NameEnc = encode_bin_elems(Name), 370 | RestEnc = encode_bin_elems(Rest), 371 | <>. 372 | 373 | encode_atom(A) -> 374 | Bin = list_to_binary(atom_to_list(A)), 375 | Enc = encode_bin_elems(Bin), 376 | <>. 377 | 378 | encode_number(N) -> 379 | encode_number(N, false). 380 | 381 | encode_number(N, Legacy) when is_integer(N) -> 382 | encode_int(N, none, Legacy); 383 | encode_number(F, _Legacy) when is_float(F) -> 384 | encode_float(F). 385 | 386 | %% 387 | %% IEEE 764 Binary 64 standard representation 388 | %% http://en.wikipedia.org/wiki/Double_precision_floating-point_format 389 | %% 390 | %% |12345678 12345678 12345678 12345678 12345678 12345678 12345678 12345678 391 | %% |iEEEEEEE EEEEffff ffffffff ffffffff ffffffff ffffffff ffffffff ffffffff| 392 | %% 393 | %% i: sign bit 394 | %% E: Exponent, 11 bits 395 | %% f: fraction, 52 bits 396 | %% 397 | %% We perform the following operations: 398 | %% - if E < 1023 (see Exponent bias), the integer part is 0 399 | %% 400 | encode_float(F) -> 401 | <> = <>, 402 | ?dbg("F = ~p | Exp0 = ~p | Frac = ~p~n", [cF, Exp0, Frac]), 403 | {Int0, Fraction} = 404 | case Exp0 - 1023 of 405 | NegExp when NegExp < 0 -> 406 | Offs = -NegExp, 407 | ?dbg("NegExp = ~p, Offs = ~p~n" 408 | "Frac = ~p~n", [NegExp, Offs, Frac]), 409 | {0, << 0:Offs, 1:1,Frac:52 >>}; 410 | Exp1 -> 411 | ?dbg("Exp1 = ~p~n", [Exp1]), 412 | if Exp1 >= 52 -> 413 | %% Decimal part will be zero 414 | {trunc(F), <<0:52>>}; 415 | true -> 416 | R = 52-Exp1, 417 | ?dbg("R = ~p~n", [R]), 418 | Exp2 = Exp1 + 1, % add the leading 1-bit 419 | ?dbg("Exp2 = ~p~n", [Exp2]), 420 | <> = <<1:1, Frac:52>>, 421 | ?dbg("I = ~p, Frac1 = ~p~n", [I,Frac1]), 422 | {I, <>} 423 | end 424 | end, 425 | if Sign == 1 -> 426 | %% explicitly encode a negative int, since Int0 can be zero. 427 | Int = if Int0 >= 0 -> -Int0; 428 | true -> Int0 429 | end, 430 | encode_neg_int(Int, Fraction); 431 | Sign == 0 -> 432 | encode_int(Int0, Fraction) 433 | end. 434 | 435 | encode_neg_int(Int, Fraction)-> 436 | encode_neg_int(Int, Fraction,false). 437 | encode_int(I, R) -> 438 | encode_int(I, R, false). 439 | 440 | encode_int(I,R, _Legacy) when I >= 0, I =< 16#7fffffff -> 441 | ?dbg("encode_int(~p, ~p)~n", [I,R]), 442 | if R == none -> 443 | << ?pos4, I:31, 0:1 >>; 444 | true -> 445 | RSz = bit_size(R), 446 | <> = R, 447 | ?dbg("Fraction = ~p~n", [Fraction]), 448 | if Fraction == 0 -> 449 | << ?pos4, I:31, 1:1, 8:8 >>; 450 | true -> 451 | Rbits = encode_bits_elems(R), 452 | << ?pos4, I:31, 1:1, Rbits/binary >> 453 | end 454 | end; 455 | encode_int(I,R, Legacy) when I > 16#7fffffff -> 456 | ?dbg("encode_int(~p, ~p)~n", [I,R]), 457 | Bytes = encode_big(I, Legacy), 458 | if R == none -> 459 | <>; 460 | true -> 461 | RSz = bit_size(R), 462 | <> = R, 463 | ?dbg("Fraction = ~p~n", [Fraction]), 464 | if Fraction == 0 -> 465 | << ?posbig, Bytes/binary, 1:8, 8:8 >>; 466 | true -> 467 | Rbits = encode_bits_elems(R), 468 | <> 469 | end 470 | end; 471 | encode_int(I, R, Legacy) when I < 0 -> 472 | encode_neg_int(I, R,Legacy). 473 | 474 | encode_neg_int(I,R,_Legacy) when I =< 0, I >= -16#7fffffff -> 475 | ?dbg("encode_neg_int(~p, ~p [sz: ~p])~n", [I,pp(R), try bit_size(R) catch error:_ -> "***" end]), 476 | Adj = max_value(31) + I, % keep in mind that I < 0 477 | ?dbg("Adj = ~p~n", [erlang:integer_to_list(Adj,2)]), 478 | if R == none -> 479 | << ?neg4, Adj:31, 1:1 >>; 480 | true -> 481 | Rbits = encode_neg_bits(R), 482 | ?dbg("R = ~p -> RBits = ~p~n", [pp(R), pp(Rbits)]), 483 | << ?neg4, Adj:31, 0:1, Rbits/binary >> 484 | end; 485 | encode_neg_int(I,R,Legacy) when I < -16#7fFFffFF -> 486 | ?dbg("encode_neg_int(BIG ~p)~n", [I]), 487 | Bytes = encode_big_neg(I,Legacy), 488 | ?dbg("Bytes = ~p~n", [Bytes]), 489 | if R == none -> 490 | <>; 491 | true -> 492 | Rbits = encode_neg_bits(R), 493 | ?dbg("R = ~p -> RBits = ~p~n", [pp(R), pp(Rbits)]), 494 | <> 495 | end. 496 | 497 | encode_big(I, Legacy) -> 498 | Bl = encode_big1(I), 499 | ?dbg("Bl = ~p~n", [Bl]), 500 | Bb = case Legacy of 501 | false -> 502 | prepend_size(list_to_binary(Bl)); 503 | true -> 504 | list_to_binary(Bl) 505 | end, 506 | ?dbg("Bb = ~p~n", [Bb]), 507 | encode_bin_elems(Bb). 508 | 509 | prepend_size(B) -> 510 | Sz = byte_size(B), 511 | <<255, (encode_size(Sz))/binary, B/binary>>. 512 | 513 | remove_size_bits(<<255, T/binary>>) -> 514 | {_, Rest} = untag_7bits(T, <<>>), 515 | Rest; 516 | remove_size_bits(B) -> 517 | %% legacy bignum 518 | B. 519 | 520 | encode_size(I) when I > 127 -> 521 | B = int_to_binary(I), 522 | tag_7bits(B); 523 | encode_size(I) -> 524 | <>. 525 | 526 | tag_7bits(B) when bit_size(B) > 7 -> 527 | <> = B, 528 | <<1:1, H:7, (tag_7bits(T))/binary>>; 529 | tag_7bits(B) -> 530 | Sz = bit_size(B), 531 | <> = B, 532 | <<0:1, I:7>>. 533 | 534 | untag_7bits(<<1:1, H:7, T/binary>>, Acc) -> 535 | untag_7bits(T, <>); 536 | untag_7bits(<<0:1, H:7, T/binary>>, Acc) -> 537 | AccBits = bit_size(Acc), 538 | HBits = 8 - (AccBits rem 8), 539 | {<>, T}. 540 | 541 | int_to_binary(I) when I =< 16#ff -> <>; 542 | int_to_binary(I) when I =< 16#ffff -> <>; 543 | int_to_binary(I) when I =< 16#ffffff -> <>; 544 | int_to_binary(I) when I =< 16#ffffffff -> <>; 545 | int_to_binary(I) when I =< 16#ffffffffff -> <>; 546 | int_to_binary(I) when I =< 16#ffffffffffff -> <>; 547 | int_to_binary(I) when I =< 16#ffffffffffffff -> <>; 548 | int_to_binary(I) when I =< 16#ffffffffffffffff -> <>; 549 | int_to_binary(I) -> 550 | %% Realm of the ridiculous 551 | list_to_binary( 552 | lists:dropwhile(fun(X) -> X==0 end, binary_to_list(<>))). 553 | 554 | %% This function exists for documentation, but not used right now. 555 | %% It's the reverse of encode_size/1, used for encoding bignums. 556 | %% 557 | %% decode_size(<<1:1, _/bitstring>> = T) -> 558 | %% {SzBin, Rest} = untag_7bits(T, <<>>), 559 | %% Bits = bit_size(SzBin), 560 | %% <> = SzBin, 561 | %% {Sz, Rest}; 562 | %% decode_size(<<0:1, H:7, T/binary>>) -> 563 | %% {H, T}. 564 | 565 | encode_big_neg(I,Legacy) -> 566 | {Words, Max} = get_max(-I), 567 | ?dbg("Words = ~p | Max = ~p~n", [Words,Max]), 568 | Iadj = Max + I, % keep in mind that I < 0 569 | ?dbg("IAdj = ~p~n", [Iadj]), 570 | Bin = encode_big(Iadj,Legacy), 571 | ?dbg("Bin = ~p~n", [Bin]), 572 | WordsAdj = 16#ffffFFFF - Words, 573 | ?dbg("WordsAdj = ~p~n", [WordsAdj]), 574 | <>. 575 | 576 | encode_big1(I) -> 577 | encode_big1(I, []). 578 | 579 | encode_big1(I, Acc) when I < 16#ff -> 580 | [I|Acc]; 581 | encode_big1(I, Acc) -> 582 | encode_big1(I bsr 8, [I band 16#ff | Acc]). 583 | 584 | encode_list_elems([], Acc, _) -> 585 | <>; 586 | encode_list_elems(B, Acc, Legacy) when is_bitstring(B) -> 587 | %% improper list 588 | <>; 589 | encode_list_elems(E, Acc, Legacy) when not(is_list(E)) -> 590 | %% improper list 591 | <>; 592 | encode_list_elems([H|T], Acc, Legacy) -> 593 | Enc = encode(H,Legacy), 594 | encode_list_elems(T, <>, Legacy). 595 | 596 | prefix_list_elems([], Acc) -> 597 | {false, <>}; 598 | prefix_list_elems(E, Acc) when not(is_list(E)) -> 599 | case is_wild(E) of 600 | true -> 601 | {true, Acc}; 602 | false -> 603 | Marker = if is_bitstring(E) -> ?bin_tail; 604 | true -> 1 605 | end, 606 | {Bool, P} = enc_prefix(E), 607 | {Bool, <>} 608 | end; 609 | prefix_list_elems([H|T], Acc) -> 610 | case enc_prefix(H) of 611 | {true, P} -> 612 | {true, <>}; 613 | {false, E} -> 614 | prefix_list_elems(T, <>) 615 | end. 616 | 617 | is_wild('_') -> 618 | true; 619 | is_wild(A) when is_atom(A) -> 620 | case atom_to_list(A) of 621 | "\$" ++ S -> 622 | try begin 623 | _ = list_to_integer(S), 624 | true 625 | end 626 | catch 627 | error:_ -> 628 | false 629 | end; 630 | _ -> 631 | false 632 | end; 633 | is_wild(_) -> 634 | false. 635 | 636 | encode_bin_elems(<<>>) -> 637 | <<8>>; 638 | encode_bin_elems(B) -> 639 | Pad = 8 - (size(B) rem 8), 640 | << (<< <<1:1, B1:8>> || <> <= B >>)/bitstring, 0:Pad, 8 >>. 641 | 642 | encode_neg_bits(<<>>) -> 643 | <<247>>; 644 | encode_neg_bits(B) -> 645 | {Padded, TailBits} = pad_neg_bytes(B), 646 | ?dbg("TailBits = ~p~n", [TailBits]), 647 | TailSz0 = bit_size(TailBits), 648 | TailSz = 16#ff - TailSz0, 649 | if TailSz0 == 0 -> 650 | Pad = 8 - (bit_size(Padded) rem 8), 651 | Ip = max_value(Pad), % e.g. max_value(3) -> 2#111 652 | <>; 653 | true -> 654 | ?dbg("TailSz0 = ~p~n", [TailSz0]), 655 | TailPad = 8 - TailSz0, 656 | ?dbg("TailPad = ~p~n", [TailPad]), 657 | Itp = (1 bsl TailPad)-1, 658 | ?dbg("Itp = ~p~n", [Itp]), 659 | Pad = 8 - ((bit_size(Padded) + 1) rem 8), 660 | ?dbg("Pad = ~p~n", [Pad]), 661 | Ip = max_value(Pad), 662 | ?dbg("Ip = ~p~n", [Ip]), 663 | ?dbg("Pad = ~p~n", [Pad]), 664 | ?dbg("TailSz = ~p~n", [TailSz]), 665 | <> 667 | end. 668 | 669 | pad_neg_bytes(Bin) -> 670 | pad_neg_bytes(Bin, <<>>). 671 | 672 | pad_neg_bytes(<>, Acc) -> 673 | H1 = 16#ff - H, 674 | pad_neg_bytes(T, <>); 675 | pad_neg_bytes(Bits, Acc) when is_bitstring(Bits) -> 676 | Sz = bit_size(Bits), 677 | Max = (1 bsl Sz) - 1, 678 | <> = Bits, 679 | I1 = Max - I0, 680 | {Acc, <>}. 681 | 682 | encode_bits_elems(B) -> 683 | {Padded, TailBits} = pad_bytes(B), 684 | TailSz = bit_size(TailBits), 685 | TailPad = 8-TailSz, 686 | Pad = 8 - ((TailSz + TailPad + bit_size(Padded) + 1) rem 8), 687 | <>. 688 | 689 | pad_bytes(Bin) -> 690 | pad_bytes(Bin, <<>>). 691 | 692 | pad_bytes(<>, Acc) -> 693 | pad_bytes(T, <>); 694 | pad_bytes(Bits, Acc) when is_bitstring(Bits) -> 695 | {Acc, Bits}. 696 | 697 | 698 | %% ------------------------------------------------------ 699 | %% Decoding routines 700 | 701 | -spec decode_next(binary()) -> {any(), binary()}. 702 | %% @spec decode_next(Bin) -> {N, Rest} 703 | %% @doc Decode a binary stream, returning the next decoded term and the 704 | %% stream remainder 705 | %% 706 | %% This function will raise an exception if the beginning of `Bin' is not 707 | %% a valid sext-encoded term. 708 | %% @end 709 | decode_next(<>) -> decode_atom(Rest); 710 | decode_next(<>) -> decode_pid(Rest); 711 | decode_next(<>) -> decode_port(Rest); 712 | decode_next(<>) -> decode_ref(Rest); 713 | decode_next(<>) -> decode_tuple(Sz,Rest); 714 | decode_next(<>) -> decode_list(Rest); 715 | decode_next(<>) -> decode_neg_big(Rest); 716 | decode_next(<>) -> decode_pos_big(Rest); 717 | decode_next(<>) -> decode_neg(I,F,Rest); 718 | decode_next(<>) -> decode_pos(I,F,Rest); 719 | decode_next(<>) -> decode_binary(Rest). 720 | 721 | -spec partial_decode(binary()) -> {full | partial, any(), binary()}. 722 | %% @spec partial_decode(Bytes) -> {full | partial, DecodedTerm, Rest} 723 | %% @doc Decode a sext-encoded term or prefix embedded in a byte stream. 724 | %% 725 | %% Example: 726 | %% ``` 727 | %% 1> T = sext:encode({a,b,c}). 728 | %% <<16,0,0,0,3,12,176,128,8,12,177,0,8,12,177,128,8>> 729 | %% 2> sext:partial_decode(<<T/binary, "tail">>). 730 | %% {full,{a,b,c},<<"tail">>} 731 | %% 3> P = sext:prefix({a,b,'_'}). 732 | %% <<16,0,0,0,3,12,176,128,8,12,177,0,8>> 733 | %% 4> sext:partial_decode(<<P/binary, "tail">>). 734 | %% {partial,{a,b,'_'},<<"tail">>} 735 | %% ''' 736 | %% 737 | %% Note that a decoded prefix may not be exactly like the encoded prefix. 738 | %% For example, ['_'] will be encoded as 739 | %% <<17>>, i.e. only the 'list' opcode. The 740 | %% decoded prefix will be '_', since the encoded prefix would 741 | %% also match the empty list. The decoded prefix will always be a prefix to 742 | %% anything to which the original prefix is a prefix. 743 | %% 744 | %% For tuples, {1,'_',3} encoded and decoded, will result in 745 | %% {1,'_','_'}, i.e. the tuple size is kept, but the elements 746 | %% after the first wildcard are replaced with wildcards. 747 | %% @end 748 | partial_decode(<>) -> 749 | partial_decode_tuple(Sz, Rest); 750 | partial_decode(<>) -> 751 | partial_decode_list(Rest); 752 | partial_decode(Other) -> 753 | try decode_next(Other) of 754 | {Dec, Rest} -> 755 | {full, Dec, Rest} 756 | catch 757 | error:function_clause -> 758 | {partial, '_', Other} 759 | end. 760 | 761 | decode_atom(B) -> 762 | {Bin, Rest} = decode_binary(B), 763 | {list_to_atom(binary_to_list(Bin)), Rest}. 764 | 765 | decode_tuple(Sz, Elems) -> 766 | decode_tuple(Sz,Elems,[]). 767 | 768 | decode_tuple(0, Rest, Acc) -> 769 | {list_to_tuple(lists:reverse(Acc)), Rest}; 770 | decode_tuple(N, Elems, Acc) -> 771 | {Term, Rest} = decode_next(Elems), 772 | decode_tuple(N-1, Rest, [Term|Acc]). 773 | 774 | partial_decode_tuple(Sz, Elems) -> 775 | partial_decode_tuple(Sz, Elems, []). 776 | 777 | partial_decode_tuple(0, Rest, Acc) -> 778 | {full, list_to_tuple(lists:reverse(Acc)), Rest}; 779 | partial_decode_tuple(N, Elems, Acc) -> 780 | case partial_decode(Elems) of 781 | {partial, Term, Rest} -> 782 | {partial, list_to_tuple( 783 | lists:reverse([Term|Acc]) ++ pad_(N-1)), Rest}; 784 | {full, Dec, Rest} -> 785 | partial_decode_tuple(N-1, Rest, [Dec|Acc]) 786 | end. 787 | 788 | pad_(0) -> 789 | []; 790 | pad_(N) when N > 0 -> 791 | ['_'|pad_(N-1)]. 792 | 793 | partial_decode_list(Elems) -> 794 | partial_decode_list(Elems, []). 795 | 796 | partial_decode_list(<<>>, Acc) -> 797 | {partial, lists:reverse(Acc) ++ '_', <<>>}; 798 | partial_decode_list(<<2, Rest/binary>>, Acc) -> 799 | {full, lists:reverse(Acc), Rest}; 800 | partial_decode_list(<>, Acc) -> 801 | %% improper list, binary tail 802 | {Term, Rest} = decode_next(Next), 803 | {full, lists:reverse(Acc) ++ Term, Rest}; 804 | partial_decode_list(<<1, Next/binary>>, Acc) -> 805 | {Result, Term, Rest} = partial_decode(Next), 806 | {Result, lists:reverse(Acc) ++ Term, Rest}; 807 | partial_decode_list(<> = Next, Acc) when ?is_sext(X) -> 808 | case partial_decode(Next) of 809 | {full, Term, Rest} -> 810 | partial_decode_list(Rest, [Term|Acc]); 811 | {partial, Term, Rest} -> 812 | {partial, lists:reverse([Term|Acc]) ++ '_', Rest} 813 | end; 814 | partial_decode_list(Rest, Acc) -> 815 | {partial, lists:reverse(Acc) ++ '_', Rest}. 816 | 817 | decode_list(Elems) -> 818 | decode_list(Elems, []). 819 | 820 | decode_list(<<2, Rest/binary>>, Acc) -> 821 | {lists:reverse(Acc), Rest}; 822 | decode_list(<>, Acc) -> 823 | %% improper list, binary tail 824 | {Term, Rest} = decode_next(Next), 825 | {lists:reverse(Acc) ++ Term, Rest}; 826 | decode_list(<<1, Next/binary>>, Acc) -> 827 | %% improper list, non-binary tail 828 | {Term, Rest} = decode_next(Next), 829 | {lists:reverse(Acc) ++ Term, Rest}; 830 | decode_list(Elems, Acc) -> 831 | {Term, Rest} = decode_next(Elems), 832 | decode_list(Rest, [Term|Acc]). 833 | 834 | decode_pid(Bin) -> 835 | {Name, Rest} = decode_binary(Bin), 836 | NameSz = size(Name), 837 | case Rest of 838 | <> -> 839 | {binary_to_term(<<131,88,100,NameSz:16,Name/binary,NS/binary,C/binary>>), Rest1}; 840 | <> -> 841 | true = C =< 3, 842 | {binary_to_term(<<131,103,100,NameSz:16,Name/binary,NS/binary,C>>), Rest1} 843 | end. 844 | 845 | decode_port(Bin) -> 846 | {Name, Rest} = decode_binary(Bin), 847 | NameSz = size(Name), 848 | case Rest of 849 | <<16, N:8/binary, 255, C:4/binary, Rest1/binary>> -> 850 | {binary_to_term(<<131,120,100,NameSz:16,Name/binary,N/binary,C/binary>>), Rest1}; 851 | <> -> 852 | {binary_to_term(<<131,89,100,NameSz:16,Name/binary,N/binary,C/binary>>), Rest1}; 853 | <> -> 854 | true = C =< 3, 855 | {binary_to_term(<<131,102,100,NameSz:16,Name/binary,N/binary,C>>), Rest1} 856 | end. 857 | 858 | decode_ref(Bin) -> 859 | {Name, Rest} = decode_binary(Bin), 860 | {Tail, Rest1} = decode_binary(Rest), 861 | NLen = size(Name), 862 | case Tail of 863 | <<255, C:4/binary, Tail1/binary>> -> 864 | Len = size(Tail1) div 4, 865 | RefBin = <<131,90,Len:16,100,NLen:16,Name/binary,C/binary,Tail1/binary>>, 866 | {binary_to_term(RefBin), Rest1}; 867 | <> -> 868 | true = C =< 3, 869 | Len = size(Tail1) div 4, 870 | RefBin = <<131,114,Len:16,100,NLen:16,Name/binary,C,Tail1/binary>>, 871 | {binary_to_term(RefBin), Rest1} 872 | end. 873 | 874 | decode_neg(I, 1, Rest) -> 875 | {(I - 16#7fffFFFF), Rest}; 876 | decode_neg(I0, 0, Bin) -> % for negative numbers, 0 means that it's a float 877 | I = 16#7fffFFFF - I0, 878 | ?dbg("decode_neg()... I = ~p | Bin = ~p~n", [I, Bin]), 879 | decode_neg_float(I, Bin). 880 | 881 | decode_neg_float(0, Bin) -> 882 | {R, Rest} = decode_neg_binary(Bin), 883 | ?dbg("Bin = ~p~n", [pp(Bin)]), 884 | ?dbg("R = ~p | Rest = ~p~n", [pp(R), Rest]), 885 | Sz = bit_size(R), 886 | Offs = Sz - 53, 887 | ?dbg("Offs = ~p | Sz - ~p~n", [Offs, Sz]), 888 | <<_:Offs, 1:1, I:52>> = R, 889 | Exp = 1023 - Offs, 890 | <> = <<1:1, Exp:11, I:52>>, 891 | {F, Rest}; 892 | decode_neg_float(I, Bin) -> 893 | {R, Rest} = decode_neg_binary(Bin), 894 | ?dbg("decode_neg_float: I = ~p | R = ~p~n", [I, R]), 895 | Sz = bit_size(R), 896 | ?dbg("Sz = ~p~n", [Sz]), 897 | <> = R, 898 | ?dbg("Ri = ~p~n", [Ri]), 899 | if Ri == 0 -> 900 | %% special case 901 | {0.0-I, Rest}; 902 | true -> 903 | IBits = strip_first_one(I), 904 | ?dbg("IBits = ~p~n", [pp(IBits)]), 905 | Bits = <>, 906 | ?dbg("Bits = ~p (Sz: ~p)~n", [pp(Bits), bit_size(Bits)]), 907 | Exp = bit_size(IBits) + 1023, 908 | ?dbg("Exp = ~p~n", [Exp]), 909 | <> = <>, 910 | ?dbg("Frac = ~p~n", [Frac]), 911 | <> = <<1:1, Exp:11, Frac:52>>, 912 | {F, Rest} 913 | end. 914 | 915 | decode_pos(I, 0, Rest) -> 916 | {I, Rest}; 917 | decode_pos(0, 1, Bin) -> 918 | {Real, Rest} = decode_binary(Bin), 919 | Offs = bit_size(Real) - 53, 920 | <<0:Offs, 1:1, Frac:52>> = Real, 921 | Exp = 1023 - Offs, 922 | <> = <<0:1, Exp:11, Frac:52>>, 923 | {F, Rest}; 924 | decode_pos(I, 1, Bin) -> % float > 1 925 | ?dbg("decode_pos(~p, 1, ~p)~n", [I, Bin]), 926 | {Real, Rest} = decode_binary(Bin), 927 | case decode_binary(Bin) of 928 | {<<>>, Rest} -> 929 | <> = <>, 930 | {F, Rest}; 931 | {Real, Rest} -> 932 | ?dbg("Real = ~p~n", [Real]), 933 | Exp = 52 - bit_size(Real) + 1023, 934 | ?dbg("Exp = ~p~n", [Exp]), 935 | Bits0 = <>, 936 | ?dbg("Bits0 = ~p~n", [Bits0]), 937 | Bits = strip_one(Bits0), 938 | <> = Bits, 939 | <> = <<0:1, Exp:11, Frac:52>>, 940 | {F, Rest} 941 | end. 942 | 943 | decode_pos_big(Bin) -> 944 | ?dbg("decode_pos_big(~p)~n", [Bin]), 945 | {Ib0, Rest} = decode_binary(Bin), 946 | Ib = remove_size_bits(Ib0), 947 | ?dbg("Ib = ~p~n", [Ib]), 948 | ISz = size(Ib) * 8, 949 | ?dbg("ISz = ~p~n", [ISz]), 950 | <> = Ib, 951 | ?dbg("I = ~p~n", [I]), 952 | <> = Rest, 953 | ?dbg("Rest1 = ~p~n", [Rest1]), 954 | decode_pos(I, F, Rest1). 955 | 956 | decode_neg_big(Bin) -> 957 | ?dbg("decode_neg_big(~p)~n", [Bin]), 958 | <> = Bin, 959 | Words = 16#ffffFFFF - WordsAdj, 960 | ?dbg("Words = ~p~n", [Words]), 961 | {Ib0, Rest1} = decode_binary(Rest), 962 | Ib = remove_size_bits(Ib0), 963 | ?dbg("Ib = ~p | Rest1 = ~p~n", [Ib, Rest1]), 964 | ISz = size(Ib) * 8, 965 | <> = Ib, 966 | ?dbg("I0 = ~p~n", [I0]), 967 | Max = imax(Words), 968 | ?dbg("Max = ~p~n", [Max]), 969 | I = Max - I0, 970 | ?dbg("I = ~p~n", [I]), 971 | <> = Rest1, 972 | ?dbg("F = ~p | Rest2 = ~p~n", [F, Rest2]), 973 | if F == 0 -> 974 | decode_neg_float(I, Rest2); 975 | F == 16#ff -> 976 | {-I, Rest2} 977 | end. 978 | 979 | %% optimization - no need to loop through a very large number of zeros. 980 | strip_first_one(I) -> 981 | Sz = if I < 16#ff -> 8; 982 | I < 16#ffff -> 16; 983 | I < 16#ffffff -> 24; 984 | I < 16#ffffffff -> 32; 985 | true -> 52 986 | end, 987 | strip_one(<>). 988 | 989 | strip_one(<<0:1, Rest/bitstring>>) -> strip_one(Rest); 990 | strip_one(<<1:1, Rest/bitstring>>) -> Rest. 991 | 992 | 993 | decode_binary(<<8, Rest/binary>>) -> {<<>>, Rest}; 994 | decode_binary(B) -> decode_binary(B, 0, <<>>). 995 | 996 | decode_binary(<<1:1,H:8,Rest/bitstring>>, N, Acc) -> 997 | case Rest of 998 | <<1:1,_/bitstring>> -> 999 | decode_binary(Rest, N+9, << Acc/binary, H >>); 1000 | _ -> 1001 | Pad = 8 - ((N+9) rem 8), 1002 | <<0:Pad,EndBits,Rest1/binary>> = Rest, 1003 | TailPad = 8-EndBits, 1004 | <> = <>, 1005 | {<< Acc/binary, Tail:EndBits >>, Rest1} 1006 | end. 1007 | 1008 | decode_neg_binary(<<247, Rest/binary>>) -> {<<>>, Rest}; % 16#ff - 8 1009 | decode_neg_binary(B) -> decode_neg_binary(B, 0, <<>>). 1010 | 1011 | decode_neg_binary(<<0:1,H:8,Rest/bitstring>>, N, Acc) -> 1012 | case Rest of 1013 | <<0:1,_/bitstring>> -> 1014 | decode_neg_binary(Rest, N+9, << Acc/binary, (16#ff - H) >>); 1015 | _ -> 1016 | Pad = 8 - ((N+9) rem 8), 1017 | ?dbg("Pad = ~p~n", [Pad]), 1018 | IPad = (1 bsl Pad) - 1, 1019 | <> = Rest, 1020 | ?dbg("EndBits0 = ~p~n", [EndBits0]), 1021 | EndBits = 16#ff - EndBits0, 1022 | ?dbg("EndBits = ~p~n", [EndBits]), 1023 | if EndBits == 0 -> 1024 | {<< Acc/binary, (16#ff - H)>>, Rest1}; 1025 | true -> 1026 | <> = <<(16#ff - H)>>, 1027 | ?dbg("Tail = ~p~n", [Tail]), 1028 | {<< Acc/binary, Tail:EndBits >>, Rest1} 1029 | end 1030 | end. 1031 | 1032 | %% The largest value that fits in Sz bits 1033 | max_value(Sz) -> 1034 | (1 bsl Sz) - 1. 1035 | 1036 | %% The largest value that fits in Words*64 bits. 1037 | imax(1) -> max_value(64); 1038 | imax(2) -> max_value(128); 1039 | imax(Words) -> max_value(Words*64). 1040 | 1041 | %% Get the smallest imax/1 value that's larger than I. 1042 | get_max(I) -> get_max(I, 1, imax(1)). 1043 | get_max(I, W, Max) when I > Max -> 1044 | get_max(I, W+1, (Max bsl 64) bor ?IMAX1); 1045 | get_max(_, W, Max) -> 1046 | {W, Max}. 1047 | 1048 | %% @spec to_sb32(Bits::bitstring()) -> binary() 1049 | %% @doc Converts a bitstring into an sb-encoded bitstring 1050 | %% 1051 | %% sb32 (Sortable base32) is a variant of RFC3548, slightly rearranged to 1052 | %% preserve the lexical sorting properties. Base32 was chosen to avoid 1053 | %% filename-unfriendly characters. Also important is that the padding 1054 | %% character be less than any character in the alphabet 1055 | %% 1056 | %% sb32 alphabet: 1057 | %% 
1058 | %% 0 0     6 6     12 C     18 I     24 O     30 U
1059 | %% 1 1     7 7     13 D     19 J     25 P     31 V
1060 | %% 2 2     8 8     14 E     20 K     26 Q  (pad) -
1061 | %% 3 3     9 9     15 F     21 L     27 R
1062 | %% 4 4    10 A     16 G     22 M     28 S
1063 | %% 5 5    11 B     17 H     23 N     29 T
1064 | %%
1065 | %% @end 1066 | %% 1067 | to_sb32(Bits) when is_bitstring(Bits) -> 1068 | Sz = bit_size(Bits), 1069 | {Chunk, Rest, Pad} = 1070 | case Sz rem 5 of 1071 | 0 -> {Bits, <<>>, <<>>}; 1072 | R -> sb32_encode_chunks(Sz, R, Bits) 1073 | end, 1074 | Enc = << << (c2sb32(C1)) >> || 1075 | <> <= Chunk >>, 1076 | if Rest == << >> -> 1077 | Enc; 1078 | true -> 1079 | << Enc/bitstring, (c2sb32(Rest)):8, Pad/binary >> 1080 | end. 1081 | 1082 | sb32_encode_chunks(Sz, Rem, Bits) -> 1083 | ChunkSz = Sz - Rem, 1084 | << C:ChunkSz/bitstring, Rest:Rem >> = Bits, 1085 | Pad = encode_pad(Rem), 1086 | {C, Rest, Pad}. 1087 | 1088 | encode_pad(3) -> <<"------">>; 1089 | encode_pad(1) -> <<"----">>; 1090 | encode_pad(4) -> <<"---">>; 1091 | encode_pad(2) -> <<"-">>. 1092 | 1093 | %% @spec from_sb32(Bits::bitstring()) -> bitstring() 1094 | %% @doc Converts from an sb32-encoded bitstring into a 'normal' bitstring 1095 | %% 1096 | %% This function is the reverse of {@link to_sb32/1}. 1097 | %% @end 1098 | %% 1099 | from_sb32(<< C:8, "------" >>) -> << (sb322c(C)):3 >>; 1100 | from_sb32(<< C:8, "----" >> ) -> << (sb322c(C)):1 >>; 1101 | from_sb32(<< C:8, "---" >> ) -> << (sb322c(C)):4 >>; 1102 | from_sb32(<< C:8, "-" >> ) -> << (sb322c(C)):2 >>; 1103 | from_sb32(<< C:8, Rest/bitstring >>) -> 1104 | << (sb322c(C)):5, (from_sb32(Rest))/bitstring >>; 1105 | from_sb32(<< >>) -> 1106 | << >>. 1107 | 1108 | c2sb32(I) when 0 =< I, I =< 9 -> $0 + I; 1109 | c2sb32(I) when 10 =< I, I =< 31 -> $A + I - 10. 1110 | 1111 | sb322c(I) when $0 =< I, I =< $9 -> I - $0; 1112 | sb322c(I) when $A =< I, I =< $V -> I - $A + 10. 1113 | 1114 | %% @spec to_hex(Bin::binary()) -> binary() 1115 | %% @doc Converts a binary into a hex-encoded binary 1116 | %% This is conventional hex encoding, with the proviso that 1117 | %% only capital letters are used, e.g. `0..9A..F'. 1118 | %% @end 1119 | to_hex(Bin) -> 1120 | << << (nib2hex(N)):8 >> || <> <= Bin >>. 1121 | 1122 | %% @spec from_hex(Bin::binary()) -> binary() 1123 | %% @doc Converts from a hex-encoded binary into a 'normal' binary 1124 | %% 1125 | %% This function is the reverse of {@link to_hex/1}. 1126 | %% 1127 | from_hex(Bin) -> 1128 | << << (hex2nib(H)):4 >> || <> <= Bin >>. 1129 | 1130 | nib2hex(N) when 0 =< N, N =< 9 -> $0 + N; 1131 | nib2hex(N) when 10 =< N, N =< 15-> $A + N - 10. 1132 | 1133 | hex2nib(C) when $0 =< C, C =< $9 -> C - $0; 1134 | hex2nib(C) when $A =< C, C =< $F -> C - $A + 10. 1135 | 1136 | -ifdef(DEBUG). 1137 | pp(none) -> ""; 1138 | pp(B) when is_bitstring(B) -> 1139 | [ $0 + I || <> <= B ]. 1140 | -endif. 1141 | 1142 | -ifdef(TEST). 1143 | -include_lib("eunit/include/eunit.hrl"). 1144 | 1145 | encode_test() -> 1146 | L = test_list(), 1147 | [{I,I} = {I,catch decode(encode(I))} || I <- L]. 1148 | 1149 | test_list() -> 1150 | [-456453453477456464.45456, 1151 | -5.23423564, 1152 | -1.234234, 1153 | -1.23423, 1154 | -0.345, 1155 | -0.34567, 1156 | -0.0034567, 1157 | 0, 1158 | 0.00012345, 1159 | 0.12345, 1160 | 1.2345, 1161 | 123.45, 1162 | 456453453477456464.45456, 1163 | a, 1164 | aaa, 1165 | {}, 1166 | {1}, 1167 | {1,2}, 1168 | {"","123"}, 1169 | {"1","234"}, 1170 | <<>>, 1171 | <<1>>, 1172 | <<1,5:3>>, 1173 | <<1,5:4>>, 1174 | [1,2,3], 1175 | [], 1176 | self(), 1177 | spawn(fun() -> ok end), 1178 | make_ref(), 1179 | make_ref()| 1180 | lists:sublist(erlang:ports(),1,2)]. 1181 | 1182 | -endif. 1183 | -------------------------------------------------------------------------------- /src/mnesia_eleveldb_sup.erl: -------------------------------------------------------------------------------- 1 | %%---------------------------------------------------------------- 2 | %% Copyright (c) 2013-2016 Klarna AB 3 | %% 4 | %% This file is provided to you under the Apache License, 5 | %% Version 2.0 (the "License"); you may not use this file 6 | %% except in compliance with the License. You may obtain 7 | %% a copy of the License at 8 | %% 9 | %% http://www.apache.org/licenses/LICENSE-2.0 10 | %% 11 | %% Unless required by applicable law or agreed to in writing, 12 | %% software distributed under the License is distributed on an 13 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | %% KIND, either express or implied. See the License for the 15 | %% specific language governing permissions and limitations 16 | %% under the License. 17 | %%---------------------------------------------------------------- 18 | 19 | -module(mnesia_eleveldb_sup). 20 | 21 | -behaviour(supervisor). 22 | 23 | %% API 24 | -export([start_link/0]). 25 | 26 | %% Supervisor callbacks 27 | -export([init/1]). 28 | 29 | %% Helper macro for declaring children of supervisor 30 | -define(CHILD(I, Type), {I, {I, start_link, []}, permanent, 5000, Type, [I]}). 31 | 32 | %% =================================================================== 33 | %% API functions 34 | %% =================================================================== 35 | 36 | start_link() -> 37 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 38 | 39 | %% =================================================================== 40 | %% Supervisor callbacks 41 | %% =================================================================== 42 | 43 | init([]) -> 44 | {ok, { {one_for_one, 5, 10}, [?CHILD(mnesia_eleveldb_params, worker)]} }. 45 | -------------------------------------------------------------------------------- /src/mnesia_eleveldb_tuning.erl: -------------------------------------------------------------------------------- 1 | %%---------------------------------------------------------------- 2 | %% Copyright (c) 2013-2016 Klarna AB 3 | %% 4 | %% This file is provided to you under the Apache License, 5 | %% Version 2.0 (the "License"); you may not use this file 6 | %% except in compliance with the License. You may obtain 7 | %% a copy of the License at 8 | %% 9 | %% http://www.apache.org/licenses/LICENSE-2.0 10 | %% 11 | %% Unless required by applicable law or agreed to in writing, 12 | %% software distributed under the License is distributed on an 13 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | %% KIND, either express or implied. See the License for the 15 | %% specific language governing permissions and limitations 16 | %% under the License. 17 | %%---------------------------------------------------------------- 18 | 19 | -module(mnesia_eleveldb_tuning). 20 | 21 | -export([describe_env/0, 22 | get_maxfiles/0, get_maxfiles/1, 23 | get_avail_ram/0, 24 | ldb_tabs/0, ldb_tabs/1, 25 | ldb_indexes/0, ldb_indexes/1, 26 | count_ldb_tabs/0, count_ldb_tabs/1, 27 | calc_sizes/0, calc_sizes/1, 28 | ideal_max_files/0, ideal_max_files/1, 29 | max_files/1, 30 | write_buffer/1, 31 | cache/1, 32 | default/1]). 33 | 34 | -include("mnesia_eleveldb_tuning.hrl"). 35 | 36 | -define(KB, 1024). 37 | -define(MB, 1024 * 1024). 38 | -define(GB, 1024 * 1024 * 1024). 39 | 40 | describe_env() -> 41 | #tuning{max_files = get_maxfiles(), 42 | avail_ram = get_avail_ram()}. 43 | 44 | get_maxfiles() -> 45 | get_maxfiles(os:type()). 46 | 47 | get_maxfiles({unix,darwin}) -> 48 | Limit = os:cmd("launchctl limit maxfiles"), 49 | [_, Soft, _] = string:tokens(Limit, " \t\n"), 50 | list_to_integer(Soft); 51 | get_maxfiles({unix,linux}) -> 52 | [L|_] = string:tokens(os:cmd("ulimit -n"), "\n"), 53 | list_to_integer(L). 54 | 55 | %% Returns installed RAM in Gigabytes 56 | get_avail_ram() -> 57 | get_avail_ram(os:type()). 58 | 59 | get_avail_ram({unix,darwin}) -> 60 | {match, [S]} = 61 | re:run(os:cmd("/usr/sbin/system_profiler SPHardwareDataType"), 62 | "Memory: (.+) GB", [{capture, [1], list}]), 63 | list_to_integer(S); 64 | get_avail_ram({unix,linux}) -> 65 | {match, [S]} = 66 | re:run(os:cmd("free -g"), "Mem:[ ]+([0-9]+) ",[{capture,[1],list}]), 67 | list_to_integer(S). 68 | 69 | ldb_tabs() -> 70 | ldb_tabs(mnesia_lib:dir()). 71 | 72 | ldb_tabs(Db) -> 73 | ldb_tabs(list_dir(Db), Db). 74 | 75 | ldb_tabs(Fs, _Db) -> 76 | lists:flatmap( 77 | fun(F) -> 78 | case re:run(F, "(.+)-_tab\\.extldb", 79 | [global,{capture,[1],list}]) of 80 | {match, [Match]} -> 81 | Match; 82 | _ -> 83 | [] 84 | end 85 | end, Fs). 86 | 87 | ldb_indexes() -> 88 | ldb_indexes(mnesia_lib:dir()). 89 | 90 | ldb_indexes(Db) -> 91 | ldb_indexes(list_dir(Db), Db). 92 | 93 | ldb_indexes(Fs, _Db) -> 94 | lists:flatmap( 95 | fun(F) -> 96 | case re:run(F, "(.+)-([0-9]+)-_ix\\.extldb", 97 | [global,{capture,[1,2],list}]) of 98 | {match, [[T,P]]} -> 99 | [{T,P}]; 100 | _ -> 101 | [] 102 | end 103 | end, Fs). 104 | 105 | list_dir(D) -> 106 | case file:list_dir(D) of 107 | {ok, Fs} -> Fs; 108 | {error, Reason} -> erlang:error({Reason,D}) 109 | end. 110 | 111 | fname({Tab,IxPos}, Dir) -> 112 | filename:join(Dir, Tab ++ "-" ++ IxPos ++ "-_ix.extldb"); 113 | fname(Tab, Dir) when is_list(Tab) -> 114 | filename:join(Dir, Tab ++ "-_tab.extldb"). 115 | 116 | %% Number of leveldb tables + indexes 117 | count_ldb_tabs() -> 118 | count_ldb_tabs(mnesia_lib:dir()). 119 | 120 | count_ldb_tabs(Db) -> 121 | Fs = list_dir(Db), 122 | length(ldb_tabs(Fs, Db)) + length(ldb_indexes(Fs, Db)). 123 | 124 | calc_sizes() -> 125 | calc_sizes(mnesia_lib:dir()). 126 | 127 | calc_sizes(D) -> 128 | lists:sort( 129 | fun(A,B) -> sort_size(B,A) end, % rev sort 130 | [{T, dir_size(fname(T, D))} || T <- ldb_tabs(D)] 131 | ++ [{I, dir_size(fname(I, D))} || I <- ldb_indexes(D)]). 132 | 133 | ideal_max_files() -> 134 | ideal_max_files(mnesia_lib:dir()). 135 | 136 | ideal_max_files(D) -> 137 | [{T,Sz,max_files(Sz)} || {T, Sz} <- calc_sizes(D)]. 138 | 139 | max_files({I,g}) -> 140 | round(I * 1000) div 20; 141 | max_files({I,m}) when I > 400 -> 142 | round(I) div 20; 143 | max_files(_) -> 144 | default(max_open_files). 145 | 146 | write_buffer({_,g}) -> 147 | 120 * ?MB; 148 | write_buffer({I,m}) when I > 400 -> 149 | 120 * ?MB; 150 | write_buffer(_) -> 151 | default(write_buffer). 152 | 153 | cache({_,g}) -> 154 | 8 * ?MB; 155 | cache({I,m}) when I > 400 -> 156 | 6 * ?MB; 157 | cache(_) -> 158 | default(cache). 159 | 160 | default(write_buffer) -> 2 * ?MB; 161 | default(max_open_files) -> 20; 162 | default(cache) -> 2 * ?MB. 163 | 164 | %% open_file_memory() -> 165 | %% (max_open_files-10) * 166 | %% (184 + (average_sst_filesize/2048) * 167 | %% (8 + ((average_key_size+average_value_size)/2048 +1) * 0.6). 168 | 169 | dir_size(D) -> 170 | R = os:cmd("du -hc " ++ D ++ " | grep total"), 171 | parse_sz(hd(string:tokens(R," \t\n"))). 172 | 173 | parse_sz(S) -> 174 | {match,[I,U]} = re:run(S, "([\\.0-9]+)([BKMG])", [{capture,[1,2],list}]), 175 | {scan_num(I), unit(U)}. 176 | 177 | scan_num(S) -> 178 | case erl_scan:string(S) of 179 | {ok, [{integer,_,I}],_} -> 180 | I; 181 | {ok, [{float,_,F}],_} -> 182 | F 183 | end. 184 | 185 | unit("B") -> b; 186 | unit("K") -> k; 187 | unit("M") -> m; 188 | unit("G") -> g. 189 | 190 | %% Custom sort: b < k < m < g 191 | sort_size({_,{A,U}},{_,{B,U}}) -> A < B; 192 | sort_size({_,{_,U1}},{_,{_,U2}}) -> 193 | case {U1,U2} of 194 | {b,_} -> true; 195 | {k,_} when U2 =/= b -> true; 196 | {m,g} -> true; 197 | _ -> false 198 | end. 199 | -------------------------------------------------------------------------------- /src/mnesia_eleveldb_tuning.hrl: -------------------------------------------------------------------------------- 1 | %%---------------------------------------------------------------- 2 | %% Copyright (c) 2013-2016 Klarna AB 3 | %% 4 | %% This file is provided to you under the Apache License, 5 | %% Version 2.0 (the "License"); you may not use this file 6 | %% except in compliance with the License. You may obtain 7 | %% a copy of the License at 8 | %% 9 | %% http://www.apache.org/licenses/LICENSE-2.0 10 | %% 11 | %% Unless required by applicable law or agreed to in writing, 12 | %% software distributed under the License is distributed on an 13 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | %% KIND, either express or implied. See the License for the 15 | %% specific language governing permissions and limitations 16 | %% under the License. 17 | %%---------------------------------------------------------------- 18 | 19 | -record(tuning, {max_files, 20 | n_tabs, 21 | avail_ram, 22 | files_factor = 1}). 23 | -------------------------------------------------------------------------------- /test/Emakefile: -------------------------------------------------------------------------------- 1 | mnesia_eleveldb_xform. 2 | {'*', [{parse_transform, mnesia_eleveldb_xform},debug_info]}. 3 | -------------------------------------------------------------------------------- /test/README: -------------------------------------------------------------------------------- 1 | To run the mnesia test suite, replacing disc_only_copies references with 2 | leveldb_copies: 3 | 4 | ``` 5 | cd $ERL_TOP 6 | make release_tests 7 | cd release/tests/mnesia_test 8 | cp $MNESIA_LEVELDB/test/mnesia_eleveldb_backend_xform.erl . 9 | cp $MNESIA_LEVELDB/test/Emakefile . 10 | ``` 11 | 12 | You may use github.com/uwiger/parse_trans, and pretty-print the 13 | debug_info in the transformed test suite modules using the following alias: 14 | 15 | ``` 16 | alias pp='escript $PARSE_TRANS_ROOT/ebin/parse_trans_pp.beam' 17 | ``` 18 | -------------------------------------------------------------------------------- /test/basho_bench_driver_mnesia_eleveldb.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% basho_bench: Benchmarking Suite 4 | %% 5 | %% Copyright (c) 2009-2010 Basho Techonologies 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | -module(basho_bench_driver_mnesia_eleveldb). 24 | 25 | -export([new/1, 26 | run/4]). 27 | 28 | -include("mnesia_eleveldb_basho_bench.hrl"). 29 | 30 | %% ==================================================================== 31 | %% API 32 | %% ==================================================================== 33 | 34 | new(_Id) -> 35 | Type = basho_bench_config:get(backend, ram_copies), 36 | Tab = basho_bench_config:get(mnesia_table, t), 37 | ok = bootstrap_mnesia(Tab, Type), 38 | {ok, Tab}. 39 | 40 | bootstrap_mnesia(Tab, Type) -> 41 | ok = mnesia:create_schema([node()], 42 | [{backend_types, 43 | [{leveldb_copies, mnesia_eleveldb}]}]), 44 | ok = mnesia:start(), 45 | {atomic,ok} = mnesia:create_table(Tab, [{Type, [node()]}]), 46 | mnesia:wait_for_tables([Tab], 10000). 47 | 48 | run(get, KeyGen, _ValueGen, State) -> 49 | Tab = State, 50 | Key = KeyGen(), 51 | case mnesia:dirty_read({Tab, Key}) of 52 | [] -> 53 | {ok, State}; 54 | [{_, Key, _}] -> 55 | {ok, State} 56 | end; 57 | run(put, KeyGen, ValueGen, State) -> 58 | Tab = State, 59 | ok = mnesia:dirty_write({Tab, KeyGen(), ValueGen()}), 60 | {ok, State}; 61 | run(delete, KeyGen, _ValueGen, State) -> 62 | Tab = State, 63 | ok = mnesia:dirty_delete({Tab, KeyGen()}), 64 | {ok, State}. 65 | -------------------------------------------------------------------------------- /test/mnesia_eleveldb_basho_bench.hrl: -------------------------------------------------------------------------------- 1 | 2 | -define(FAIL_MSG(Str, Args), ?ERROR(Str, Args), basho_bench_app:halt_or_kill()). 3 | -define(STD_ERR(Str, Args), io:format(standard_error, Str, Args)). 4 | 5 | -define(CONSOLE(Str, Args), lager:info(Str, Args)). 6 | 7 | -define(DEBUG(Str, Args), lager:debug(Str, Args)). 8 | -define(INFO(Str, Args), lager:info(Str, Args)). 9 | -define(WARN(Str, Args), lager:warning(Str, Args)). 10 | -define(ERROR(Str, Args), lager:error(Str, Args)). 11 | 12 | -define(FMT(Str, Args), lists:flatten(io_lib:format(Str, Args))). 13 | 14 | -define(VAL_GEN_BLOB_CFG, value_generator_blob_file). 15 | -define(VAL_GEN_SRC_SIZE, value_generator_source_size). 16 | -------------------------------------------------------------------------------- /test/mnesia_eleveldb_bench_disc_only.config: -------------------------------------------------------------------------------- 1 | {mode, max}. 2 | 3 | {duration, 10}. 4 | 5 | {concurrent, 1}. 6 | 7 | {driver, basho_bench_driver_mnesia_eleveldb}. 8 | 9 | {key_generator, {int_to_bin,{uniform_int, 5000000}}}. 10 | 11 | {value_generator, {fixed_bin, 10000}}. 12 | 13 | {operations, [{get, 2}, {put, 2}, {delete, 1}]}. 14 | 15 | {code_paths, []}. 16 | 17 | {mnesia_table, doc}. 18 | {backend, disc_only_copies}. 19 | -------------------------------------------------------------------------------- /test/mnesia_eleveldb_bench_leveldb_copies.config: -------------------------------------------------------------------------------- 1 | {mode, max}. 2 | 3 | {duration, 10}. 4 | 5 | {concurrent, 1}. 6 | 7 | {driver, basho_bench_driver_mnesia_eleveldb}. 8 | 9 | {key_generator, {int_to_bin,{uniform_int, 5000000}}}. 10 | 11 | {value_generator, {fixed_bin, 10000}}. 12 | 13 | {operations, [{get, 2}, {put, 2}, {delete, 1}]}. 14 | 15 | {code_paths, ["/Users/ulf.wiger/git/eleveldb", 16 | "/Users/ulf.wiger/git/mnesia_eleveldb"]}. 17 | 18 | {mnesia_table, ldb}. 19 | {backend, leveldb_copies}. 20 | -------------------------------------------------------------------------------- /test/mnesia_eleveldb_chg_tbl_copy.erl: -------------------------------------------------------------------------------- 1 | %%---------------------------------------------------------------- 2 | %% Copyright (c) 2013-2016 Klarna AB 3 | %% 4 | %% This file is provided to you under the Apache License, 5 | %% Version 2.0 (the "License"); you may not use this file 6 | %% except in compliance with the License. You may obtain 7 | %% a copy of the License at 8 | %% 9 | %% http://www.apache.org/licenses/LICENSE-2.0 10 | %% 11 | %% Unless required by applicable law or agreed to in writing, 12 | %% software distributed under the License is distributed on an 13 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | %% KIND, either express or implied. See the License for the 15 | %% specific language governing permissions and limitations 16 | %% under the License. 17 | %%---------------------------------------------------------------- 18 | 19 | %% @doc Run through all combinations of change_table_copy_type 20 | %% @author Ulf Wiger 21 | 22 | -module(mnesia_eleveldb_chg_tbl_copy). 23 | 24 | %% This module implements a test (to be run manually) for iterating through 25 | %% all table copy types on a mnesia table. 26 | 27 | -export([full/0, 28 | run/0, 29 | run/1]). 30 | -export([trace/0]). 31 | 32 | full() -> 33 | Perms = perms(copies()), 34 | Res = [run(P) || P <- Perms], 35 | Res = [ok || _ <- Perms]. 36 | 37 | run() -> 38 | run([ldb,disc_copies,ldb,ram_copies,disc_only_copies,ldb]). 39 | %% run([ldb,disc_only_copies]). 40 | %% run([ldb,ram_copies]). 41 | 42 | perms([]) -> 43 | [[]]; 44 | perms(L) -> [[H|T] || H <- L, T <- perms(L--[H])]. 45 | 46 | copies() -> 47 | [ldb,ram_copies,disc_copies,disc_only_copies]. 48 | 49 | run([T|Types]) -> 50 | mnesia:stop(), 51 | start_mnesia(), 52 | ok = create_tab(T), 53 | ok = change_type(Types, T). 54 | 55 | create_tab(Type) -> 56 | {atomic,ok} = mnesia:create_table( 57 | t, [{Type, [node()]}, 58 | {attributes, [k,v]}, 59 | {index, [v]}]), 60 | fill_tab(), 61 | check_tab(), 62 | ok. 63 | 64 | change_type([To|Types], From) -> 65 | io:fwrite("changing from ~p to ~p~n", [From, To]), 66 | {atomic, ok} = mnesia:change_table_copy_type(t, node(), To), 67 | ok = check_tab(), 68 | io:fwrite("...ok~n", []), 69 | change_type(Types, To); 70 | change_type([], _) -> 71 | ok. 72 | 73 | fill_tab() -> 74 | Res = [mnesia:dirty_write({t,K,V}) || {t,K,V} <- l()], 75 | Res = [ok || _ <- Res], 76 | ok. 77 | 78 | l() -> [{t,a,1}, 79 | {t,b,2}, 80 | {t,c,3}, 81 | {t,d,4}]. 82 | 83 | check_tab() -> 84 | L = l(), 85 | L = lists:append([mnesia:dirty_read({t,K}) || K <- [a,b,c,d]]), 86 | L = lists:append([mnesia:dirty_index_read(t,V,v) || 87 | V <- [1,2,3,4]]), 88 | ok. 89 | 90 | start_mnesia() -> mnesia_eleveldb_tlib:start_mnesia(reset). 91 | 92 | trace() -> 93 | dbg:tracer(), 94 | [tp(M) || M <- mods()], 95 | dbg:p(all,[c]), 96 | try run() 97 | after 98 | [ctp(M) || M <- mods()], 99 | dbg:stop() 100 | end. 101 | 102 | tp({l,M} ) -> dbg:tpl(M,x); 103 | tp({g,M} ) -> dbg:tp(M,x); 104 | tp({l,M,F}) -> dbg:tpl(M,F,x); 105 | tp({g,M,F}) -> dbg:tp(M,F,x). 106 | 107 | ctp({l,M} ) -> dbg:ctpl(M); 108 | ctp({g,M} ) -> dbg:ctp(M); 109 | ctp({l,M,F}) -> dbg:ctpl(M,F); 110 | ctp({g,M,F}) -> dbg:ctp(M,F). 111 | 112 | mods() -> 113 | [ 114 | %% {l, mnesia_index}, 115 | %% {l, mnesia_lib, semantics}]. 116 | %% {g,mnesia_monitor}, 117 | %% {l,mnesia_dumper}, 118 | %% {g,mnesia_loader}, 119 | %% {g,mnesia_checkpoint}, 120 | %% {g,mnesia_lib}, 121 | {l,mnesia_schema,expand_index_attrs}, 122 | {l,mnesia_schema,list2cs}, 123 | {g,mnesia_schema,new_cs}, 124 | {g,mnesia_schema,make_change_table_copy_type}, 125 | {g,mnesia_schema,make_create_table}, 126 | {g,mnesia_lib,semantics}, 127 | {l,mnesia_dumper}, 128 | {g,mnesia_lib,exists}, 129 | {g,mnesia}, 130 | {l,mnesia_schema,intersect_types}, 131 | {g,ets,new}]. 132 | -------------------------------------------------------------------------------- /test/mnesia_eleveldb_conv_bigtab.erl: -------------------------------------------------------------------------------- 1 | %%---------------------------------------------------------------- 2 | %% Copyright (c) 2013-2016 Klarna AB 3 | %% 4 | %% This file is provided to you under the Apache License, 5 | %% Version 2.0 (the "License"); you may not use this file 6 | %% except in compliance with the License. You may obtain 7 | %% a copy of the License at 8 | %% 9 | %% http://www.apache.org/licenses/LICENSE-2.0 10 | %% 11 | %% Unless required by applicable law or agreed to in writing, 12 | %% software distributed under the License is distributed on an 13 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | %% KIND, either express or implied. See the License for the 15 | %% specific language governing permissions and limitations 16 | %% under the License. 17 | %%---------------------------------------------------------------- 18 | 19 | -module(mnesia_eleveldb_conv_bigtab). 20 | 21 | -export([init/0, mktab/2, run/1]). 22 | 23 | -record(t, {k, i, v}). 24 | 25 | run(Sz) -> 26 | mnesia:stop(), 27 | init(), 28 | mktab(disc_copies, Sz), 29 | mnesia:change_table_copy_type(t, node(), ldb). 30 | 31 | init() -> 32 | mnesia_eleveldb_tlib:start_mnesia(reset). 33 | 34 | mktab(Backend, Sz) -> 35 | mnesia_eleveldb_tlib:create_table(Backend, [k, i, v], [i]), 36 | fill_table(Sz). 37 | 38 | 39 | fill_table(Sz) when is_integer(Sz), Sz > 0 -> 40 | fill_table(1, Sz). 41 | 42 | fill_table(N, Max) when N =< Max -> 43 | mnesia:dirty_write(#t{k = N, i = N, v = val()}), 44 | fill_table(N+1, Max); 45 | fill_table(N, _) when is_integer(N) -> 46 | ok. 47 | 48 | val() -> 49 | {1,2,3,4,5,6,7,8,9,0, 50 | 1,2,3,4,5,6,7,8,9,0, 51 | 1,2,3,4,5,6,7,8,9,0, 52 | 1,2,3,4,5,6,7,8,9,0, 53 | 1,2,3,4,5,6,7,8,9,0, 54 | 1,2,3,4,5,6,7,8,9,0, 55 | 1,2,3,4,5,6,7,8,9,0, 56 | 1,2,3,4,5,6,7,8,9,0, 57 | 1,2,3,4,5,6,7,8,9,0, 58 | 1,2,3,4,5,6,7,8,9,0, 59 | 1,2,3,4,5,6,7,8,9,0}. 60 | -------------------------------------------------------------------------------- /test/mnesia_eleveldb_fallback.erl: -------------------------------------------------------------------------------- 1 | %%---------------------------------------------------------------- 2 | %% Copyright (c) 2013-2016 Klarna AB 3 | %% 4 | %% This file is provided to you under the Apache License, 5 | %% Version 2.0 (the "License"); you may not use this file 6 | %% except in compliance with the License. You may obtain 7 | %% a copy of the License at 8 | %% 9 | %% http://www.apache.org/licenses/LICENSE-2.0 10 | %% 11 | %% Unless required by applicable law or agreed to in writing, 12 | %% software distributed under the License is distributed on an 13 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | %% KIND, either express or implied. See the License for the 15 | %% specific language governing permissions and limitations 16 | %% under the License. 17 | %%---------------------------------------------------------------- 18 | 19 | -module(mnesia_eleveldb_fallback). 20 | 21 | -export([run/0]). 22 | 23 | -define(m(A,B), fun() -> L = ?LINE, 24 | case {A,B} of 25 | {__X, __X} -> 26 | B; 27 | Other -> 28 | error({badmatch, [Other, 29 | {line, L}]}) 30 | end 31 | end()). 32 | 33 | run() -> 34 | cleanup(), 35 | mnesia_eleveldb_tlib:start_mnesia(reset), 36 | mnesia_eleveldb_tlib:create_table(ldb), 37 | ok = mnesia:backup("bup0.BUP"), 38 | [mnesia:dirty_write({t,K,V}) || {K,V} <- [{a,1}, 39 | {b,2}, 40 | {c,3}]], 41 | ok = mnesia:backup("bup1.BUP"), 42 | [mnesia:dirty_write({t,K,V}) || {K,V} <- [{d,4}, 43 | {e,5}, 44 | {f,6}]], 45 | ok = mnesia:backup("bup2.BUP"), 46 | io:fwrite("*****************************************~n", []), 47 | load_backup("bup0.BUP"), 48 | ?m([], mnesia:dirty_match_object(t, {t,'_','_'})), 49 | ?m([], mnesia:dirty_index_read(t,2,v)), 50 | io:fwrite("*****************************************~n", []), 51 | load_backup("bup1.BUP"), 52 | ?m([{t,a,1},{t,b,2},{t,c,3}], mnesia:dirty_match_object(t, {t,'_','_'})), 53 | ?m([{t,b,2}], mnesia:dirty_index_read(t,2,v)), 54 | io:fwrite("*****************************************~n", []), 55 | load_backup("bup2.BUP"), 56 | ?m([{t,a,1},{t,b,2},{t,c,3}, 57 | {t,d,4},{t,e,5},{t,f,6}], mnesia:dirty_match_object(t, {t,'_','_'})), 58 | ?m([{t,b,2}], mnesia:dirty_index_read(t,2,v)), 59 | ?m([{t,e,5}], mnesia:dirty_index_read(t,5,v)), 60 | ok. 61 | 62 | load_backup(BUP) -> 63 | mnesia_eleveldb_tlib:trace(fun() -> 64 | io:fwrite("loading backup ~s~n", [BUP]), 65 | ok = mnesia:install_fallback(BUP), 66 | io:fwrite("stopping~n", []), 67 | mnesia:stop(), 68 | timer:sleep(3000), 69 | io:fwrite("starting~n", []), 70 | mnesia:start(), 71 | WaitRes = mnesia:wait_for_tables([t], 5000), 72 | io:fwrite("WaitRes = ~p~n", [WaitRes]) 73 | end, 74 | mods(0) 75 | ). 76 | 77 | cleanup() -> 78 | os:cmd("rm *.BUP"). 79 | 80 | mods(0) -> 81 | []; 82 | mods(1) -> 83 | [ 84 | {l, mnesia_eleveldb}, 85 | {g, eleveldb} 86 | ]; 87 | mods(2) -> 88 | [ 89 | %% {l, mnesia_monitor}, 90 | {g, mnesia_eleveldb}, 91 | {l, mnesia_bup}, 92 | {g, mnesia_lib}, 93 | {g, mnesia_schema}, 94 | %% {g, mnesia_loader}, 95 | {g, mnesia_index}, 96 | {l, mnesia_tm} 97 | ]. 98 | -------------------------------------------------------------------------------- /test/mnesia_eleveldb_indexes.erl: -------------------------------------------------------------------------------- 1 | %%---------------------------------------------------------------- 2 | %% Copyright (c) 2013-2016 Klarna AB 3 | %% 4 | %% This file is provided to you under the Apache License, 5 | %% Version 2.0 (the "License"); you may not use this file 6 | %% except in compliance with the License. You may obtain 7 | %% a copy of the License at 8 | %% 9 | %% http://www.apache.org/licenses/LICENSE-2.0 10 | %% 11 | %% Unless required by applicable law or agreed to in writing, 12 | %% software distributed under the License is distributed on an 13 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | %% KIND, either express or implied. See the License for the 15 | %% specific language governing permissions and limitations 16 | %% under the License. 17 | %%---------------------------------------------------------------- 18 | 19 | -module(mnesia_eleveldb_indexes). 20 | 21 | -export([run/0, 22 | r1/0]). 23 | 24 | run() -> 25 | mnesia:stop(), 26 | ok = mnesia_eleveldb_tlib:start_mnesia(reset), 27 | test(1, ram_copies, r1), 28 | test(1, disc_copies, d1), 29 | fail(test, [1, disc_only_copies, do1]), % doesn't support ordered 30 | test(2, disc_only_copies, do1), 31 | fail(test, [1, ldb, l1]), % doesn't support bag 32 | test(3, ldb, l1), 33 | add_del_indexes(), 34 | {atomic,ok} = mnesia_schema:add_index_plugin( 35 | {pfx},mnesia_eleveldb, ix_prefixes), 36 | test_index_plugin(pr1, ram_copies, ordered), 37 | test_index_plugin(pr2, ram_copies, bag), 38 | test_index_plugin(pd1, disc_copies, ordered), 39 | fail(test_index_plugin, [pd2, disc_only_copies, ordered]), 40 | test_index_plugin(pd2, disc_copies, bag), 41 | test_index_plugin(pl2, ldb, ordered), 42 | test_index_plugin_mgmt(), 43 | ok. 44 | 45 | r1() -> 46 | mnesia:stop(), 47 | ok = mnesia_eleveldb_tlib:start_mnesia(reset), 48 | {atomic,ok} = mnesia_schema:add_index_plugin( 49 | {pfx},mnesia_eleveldb, ix_prefixes), 50 | dbg:tracer(), 51 | dbg:tpl(mnesia_schema,x), 52 | dbg:tpl(mnesia_index,x), 53 | dbg:p(all,[c]), 54 | test_index_plugin(pd2, disc_only_copies, ordered). 55 | 56 | fail(F, Args) -> 57 | try apply(?MODULE, F, Args), 58 | error(should_fail) 59 | catch 60 | error:_ -> 61 | io:fwrite("apply(~p, ~p, ~p) -> fails as expected~n", 62 | [?MODULE, F, Args]) 63 | end. 64 | 65 | test(N, Type, T) -> 66 | {atomic, ok} = mnesia:create_table(T, [{Type,[node()]}, 67 | {attributes,[k,a,b,c]}, 68 | {index, indexes(N)}]), 69 | ok = test_index(N, T). 70 | 71 | add_del_indexes() -> 72 | {atomic, ok} = mnesia:del_table_index(r1, a), 73 | {aborted, _} = mnesia:del_table_index(r1, a), 74 | {atomic, ok} = mnesia:add_table_index(r1, a), 75 | {aborted, _} = mnesia:add_table_index(r1, a), 76 | {atomic, ok} = mnesia:del_table_index(d1, a), 77 | {atomic, ok} = mnesia:add_table_index(d1, a), 78 | {atomic, ok} = mnesia:del_table_index(do1, a), 79 | {atomic, ok} = mnesia:add_table_index(do1, a), 80 | {atomic, ok} = mnesia:del_table_index(l1, a), 81 | {atomic, ok} = mnesia:add_table_index(l1, a), 82 | io:fwrite("add_del_indexes() -> ok~n", []). 83 | 84 | test_index_plugin(Tab, Type, IxType) -> 85 | {atomic, ok} = mnesia:create_table(Tab, [{Type, [node()]}, 86 | {index, [{{pfx}, IxType}]}]), 87 | mnesia:dirty_write({Tab, "foobar", "sentence"}), 88 | mnesia:dirty_write({Tab, "yellow", "sensor"}), 89 | mnesia:dirty_write({Tab, "truth", "white"}), 90 | mnesia:dirty_write({Tab, "fulcrum", "white"}), 91 | Res1 = [{Tab, "foobar", "sentence"}, {Tab, "yellow", "sensor"}], 92 | Res2 = [{Tab, "fulcrum", "white"}, {Tab, "truth", "white"}], 93 | if IxType == bag -> 94 | Res1 = lists:sort(mnesia:dirty_index_read(Tab,<<"sen">>, {pfx})), 95 | Res2 = lists:sort(mnesia:dirty_index_read(Tab,<<"whi">>, {pfx})), 96 | [{Tab,"foobar","sentence"}] = mnesia:dirty_index_read( 97 | Tab, <<"foo">>, {pfx}); 98 | IxType == ordered -> 99 | Res1 = lists:sort(mnesia:dirty_index_read(Tab,<<"sen">>, {pfx})), 100 | Res2 = lists:sort(mnesia:dirty_index_read(Tab,<<"whi">>, {pfx})), 101 | [{Tab,"foobar","sentence"}] = mnesia:dirty_index_read( 102 | Tab, <<"foo">>, {pfx}) 103 | end, 104 | io:fwrite("test_index_plugin(~p, ~p, ~p) -> ok~n", [Tab,Type,IxType]). 105 | 106 | test_index_plugin_mgmt() -> 107 | {aborted,_} = mnesia:create_table(x, [{index,[{unknown}]}]), 108 | {aborted,_} = mnesia:create_table(x, [{index,[{{unknown},bag}]}]), 109 | {aborted,_} = mnesia:create_table(x, [{index,[{{unknown},ordered}]}]), 110 | {atomic,ok} = mnesia_schema:add_index_plugin( 111 | {t}, mnesia_eleveldb,ix_prefixes), 112 | {atomic,ok} = mnesia_schema:delete_index_plugin({t}), 113 | {aborted,{bad_type,x,_}} = 114 | mnesia:create_table(x, [{index,[{{t},ordered}]}]), 115 | %% re-add plugin 116 | {atomic,ok} = mnesia_schema:add_index_plugin( 117 | {t}, mnesia_eleveldb,ix_prefixes), 118 | {atomic,ok} = 119 | mnesia:create_table(x, [{index,[{{t},ordered}]}]), 120 | {aborted,{plugin_in_use,{t}}} = 121 | mnesia_schema:delete_index_plugin({t}). 122 | 123 | test_index(1, T) -> 124 | L2 = [{T,K,x,y,z} || K <- lists:seq(4,6)], 125 | L1 = [{T,K,a,b,c} || K <- lists:seq(1,3)], 126 | true = lists:all(fun(X) -> X == ok end, 127 | [mnesia:dirty_write(Obj) || Obj <- L1 ++ L2]), 128 | L1 = lists:sort(mnesia:dirty_index_read(T,a,a)), 129 | L1 = lists:sort(mnesia:dirty_index_read(T,a,3)), 130 | L1 = mnesia:dirty_index_read(T,b,b), 131 | L1 = lists:sort(mnesia:dirty_index_read(T,c,c)), 132 | L2 = lists:sort(mnesia:dirty_index_read(T,x,a)), 133 | L2 = lists:sort(mnesia:dirty_index_read(T,x,3)), 134 | L2 = mnesia:dirty_index_read(T,y,b), 135 | L2 = lists:sort(mnesia:dirty_index_read(T,z,c)), 136 | io:fwrite("test_index(1, ~p) -> ok~n", [T]), 137 | ok; 138 | test_index(2, T) -> 139 | L1 = [{T,K,a,b,c} || K <- lists:seq(1,3)], 140 | L2 = [{T,K,x,y,z} || K <- lists:seq(4,6)], 141 | true = lists:all(fun(X) -> X == ok end, 142 | [mnesia:dirty_write(Obj) || Obj <- L1 ++ L2]), 143 | L1 = lists:sort(mnesia:dirty_index_read(T,a,a)), 144 | L1 = lists:sort(mnesia:dirty_index_read(T,a,3)), 145 | L1 = lists:sort(mnesia:dirty_index_read(T,b,b)), 146 | L1 = lists:sort(mnesia:dirty_index_read(T,c,c)), 147 | L2 = lists:sort(mnesia:dirty_index_read(T,x,a)), 148 | L2 = lists:sort(mnesia:dirty_index_read(T,x,3)), 149 | L2 = lists:sort(mnesia:dirty_index_read(T,y,b)), 150 | L2 = lists:sort(mnesia:dirty_index_read(T,z,c)), 151 | io:fwrite("test_index(1, ~p) -> ok~n", [T]), 152 | ok; 153 | test_index(3, T) -> 154 | L2 = [{T,K,x,y,z} || K <- lists:seq(4,6)], 155 | L1 = [{T,K,a,b,c} || K <- lists:seq(1,3)], 156 | true = lists:all(fun(X) -> X == ok end, 157 | [mnesia:dirty_write(Obj) || Obj <- L1 ++ L2]), 158 | L1 = mnesia:dirty_index_read(T,a,a), 159 | L1 = mnesia:dirty_index_read(T,a,3), 160 | L1 = mnesia:dirty_index_read(T,b,b), 161 | L1 = mnesia:dirty_index_read(T,c,c), 162 | L2 = mnesia:dirty_index_read(T,x,a), 163 | L2 = mnesia:dirty_index_read(T,x,3), 164 | L2 = mnesia:dirty_index_read(T,y,b), 165 | L2 = mnesia:dirty_index_read(T,z,c), 166 | io:fwrite("test_index(1, ~p) -> ok~n", [T]), 167 | ok. 168 | 169 | indexes(1) -> 170 | [a,{b,ordered},{c,bag}]; 171 | indexes(2) -> 172 | [a,b,{c,bag}]; 173 | indexes(3) -> 174 | [a,{b,ordered},{c,ordered}]. 175 | -------------------------------------------------------------------------------- /test/mnesia_eleveldb_proper_semantics_test.erl: -------------------------------------------------------------------------------- 1 | %%---------------------------------------------------------------- 2 | %% Copyright (c) 2013-2016 Klarna AB 3 | %% 4 | %% This file is provided to you under the Apache License, 5 | %% Version 2.0 (the "License"); you may not use this file 6 | %% except in compliance with the License. You may obtain 7 | %% a copy of the License at 8 | %% 9 | %% http://www.apache.org/licenses/LICENSE-2.0 10 | %% 11 | %% Unless required by applicable law or agreed to in writing, 12 | %% software distributed under the License is distributed on an 13 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | %% KIND, either express or implied. See the License for the 15 | %% specific language governing permissions and limitations 16 | %% under the License. 17 | %%---------------------------------------------------------------- 18 | 19 | %% @doc Verify dirty vs transaction semantics against leveldb mnesia backend 20 | %% @author Ulf Wiger 21 | 22 | -module(mnesia_eleveldb_proper_semantics_test). 23 | 24 | %% This module uses the proper_statem pattern to generate random 25 | %% sequences of commands, mixing dirty and transaction operations 26 | %% (including dirty ops from within transactions). Each sequence is run 27 | %% against a disc_copies table and a leveldb_copies table, after 28 | %% which the result of each operation in the sequence is compared between 29 | %% the two runs. The postcondition is that every command in every sequence 30 | %% should yield the same value against both backends. 31 | 32 | -export([test/1, 33 | prop_seq/0]). 34 | 35 | %% statem callbacks 36 | -export([initial_state/0, 37 | command/1, 38 | precondition/2, 39 | postcondition/3, 40 | next_state/3]). 41 | 42 | %% command callbacks 43 | -export([activity/2]). 44 | 45 | -include_lib("proper/include/proper.hrl"). 46 | -include_lib("eunit/include/eunit.hrl"). 47 | 48 | -record(st, {}). 49 | -define(KEYS, [a,b,c]). 50 | 51 | basic_test_() -> 52 | {timeout, 60000, [fun() -> test(100) end]}. 53 | 54 | test(N) -> 55 | setup_mnesia(), 56 | true = proper:quickcheck(?MODULE:prop_seq(), N), 57 | ok. 58 | 59 | prop_seq() -> 60 | ?FORALL(Cmds, proper_statem:commands(?MODULE), 61 | begin 62 | setup(), 63 | {H, S, Res} = 64 | proper_statem:run_commands(?MODULE, Cmds), 65 | cleanup(), 66 | ?WHENFAIL( 67 | io:fwrite("History: ~w~n" 68 | "State : ~w~n" 69 | "Result : ~w~n", [H, S, Res]), 70 | proper:aggregate( 71 | proper_statem:command_names(Cmds), Res =:= ok)) 72 | end). 73 | 74 | %% Note that this requires the leveldb application to be in the path, 75 | %% and obviously an OTP patched with the backend plugin behavior. 76 | setup_mnesia() -> 77 | stopped = mnesia:stop(), 78 | ok = mnesia:delete_schema([node()]), 79 | ok = mnesia:create_schema([node()]), 80 | ok = mnesia:start(), 81 | {ok, leveldb_copies} = mnesia_eleveldb:register(). 82 | 83 | setup() -> 84 | {atomic,ok} = mnesia:create_table(d, [{disc_copies, [node()]}, 85 | {record_name, x}]), 86 | {atomic,ok} = mnesia:create_table(l, [{leveldb_copies, [node()]}, 87 | {record_name, x}]), 88 | ok = mnesia:wait_for_tables([d, l], 30000), 89 | ok. 90 | 91 | cleanup() -> 92 | {atomic, ok} = mnesia:delete_table(d), 93 | {atomic, ok} = mnesia:delete_table(l), 94 | ok. 95 | 96 | initial_state() -> 97 | #st{}. 98 | 99 | command(#st{}) -> 100 | ?LET(Type, type(), 101 | {call, ?MODULE, activity, [Type, sequence()]}). 102 | 103 | type() -> 104 | proper_types:oneof([async_dirty, transaction]). 105 | 106 | precondition(_, _) -> 107 | true. 108 | 109 | postcondition(_, {call,?MODULE,activity,_}, {A, B}) -> 110 | A == B; 111 | postcondition(_, _, _) -> 112 | false. 113 | 114 | next_state(St, _, _) -> 115 | St. 116 | 117 | sequence() -> 118 | proper_types:list(db_cmd()). 119 | 120 | db_cmd() -> 121 | ?LET(Type, type(), 122 | proper_types:oneof([{Type, read, key()}, 123 | {Type, write, key(), value()}, 124 | {Type, delete, key()}])). 125 | 126 | key() -> 127 | proper_types:oneof([a,b,c]). 128 | 129 | value() -> 130 | proper_types:oneof([1,2,3]). 131 | 132 | activity(Type, Seq) -> 133 | {mnesia:activity(Type, fun() -> 134 | apply_seq(Type, d, Seq) 135 | end), 136 | mnesia:activity(Type, fun() -> 137 | apply_seq(Type, l, Seq) 138 | end)}. 139 | 140 | apply_seq(Type, Tab, Seq) -> 141 | apply_seq(Type, Tab, Seq, []). 142 | 143 | apply_seq(transaction=X, Tab, [H|T], Acc) -> 144 | Res = case H of 145 | {X,read, K} -> mnesia:read(Tab, K, read); 146 | {_,read, K} -> mnesia:dirty_read(Tab,K); 147 | {X,write,K,V} -> mnesia:write(Tab, {x, K, V}, write); 148 | {_,write,K,V} -> mnesia:dirty_write(Tab, {x,K,V}); 149 | {X,delete,K} -> mnesia:delete(Tab, K, write); 150 | {_,delete,K} -> mnesia:dirty_delete(Tab,K) 151 | end, 152 | apply_seq(X, Tab, T, [Res|Acc]); 153 | apply_seq(X, Tab, [H|T], Acc) -> 154 | Res = case H of 155 | {_,read, K} -> mnesia:read(Tab, K, read); 156 | {_,write,K,V} -> mnesia:write(Tab, {x, K, V}, write); 157 | {_,delete,K} -> mnesia:delete(Tab, K, write) 158 | end, 159 | apply_seq(X, Tab, T, [Res|Acc]); 160 | apply_seq(_, _, [], Acc) -> 161 | lists:reverse(Acc). 162 | -------------------------------------------------------------------------------- /test/mnesia_eleveldb_size_info.erl: -------------------------------------------------------------------------------- 1 | %%---------------------------------------------------------------- 2 | %% Copyright (c) 2013-2016 Klarna AB 3 | %% 4 | %% This file is provided to you under the Apache License, 5 | %% Version 2.0 (the "License"); you may not use this file 6 | %% except in compliance with the License. You may obtain 7 | %% a copy of the License at 8 | %% 9 | %% http://www.apache.org/licenses/LICENSE-2.0 10 | %% 11 | %% Unless required by applicable law or agreed to in writing, 12 | %% software distributed under the License is distributed on an 13 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | %% KIND, either express or implied. See the License for the 15 | %% specific language governing permissions and limitations 16 | %% under the License. 17 | %%---------------------------------------------------------------- 18 | 19 | -module(mnesia_eleveldb_size_info). 20 | 21 | -export([run/0]). 22 | 23 | -define(m(A, B), (fun(L) -> {L,A} = {L,B} end)(?LINE)). 24 | 25 | 26 | run() -> 27 | initialize_mnesia(), 28 | test_set(), 29 | test_bag(). 30 | 31 | initialize_mnesia() -> 32 | mnesia:stop(), 33 | mnesia:delete_schema([node()]), 34 | mnesia:create_schema([node()], [{backend_types, 35 | [{leveldb_copies, mnesia_eleveldb}]}]), 36 | mnesia:start(), 37 | {atomic,ok} = mnesia:create_table(s, [{type, set}, 38 | {record_name, x}, 39 | {leveldb_copies, [node()]}]), 40 | {atomic,ok} = mnesia:create_table(b, [{type, bag}, 41 | {record_name, x}, 42 | {leveldb_copies, [node()]}]), 43 | ok. 44 | 45 | test_set() -> 46 | ?m(0, mnesia:table_info(s, size)), 47 | ?m(1, w(s, 1, a)), 48 | ?m(1, w(s, 1, b)), 49 | ?m(2, w(s, 2, c)), 50 | ?m(3, w(s, 3, d)), 51 | ?m(2, d(s, 3)), 52 | mnesia:stop(), 53 | mnesia:start(), 54 | await(s), 55 | ?m(2, mnesia:table_info(s, size)). 56 | 57 | test_bag() -> 58 | ?m(0, mnesia:table_info(b, size)), 59 | ?m(1, w(b, 1, a)), 60 | ?m(2, w(b, 1, b)), 61 | ?m(3, w(b, 2, a)), 62 | ?m(4, w(b, 2, d)), 63 | ?m(5, w(b, 2, c)), 64 | ?m(4, do(b, 2, c)), 65 | ?m(2, d(b, 2)), 66 | mnesia:stop(), 67 | mnesia:start(), 68 | await(b), 69 | ?m(2, mnesia:table_info(b, size)). 70 | 71 | w(T, K, V) -> 72 | ok = mnesia:dirty_write(T, {x, K, V}), 73 | mnesia:table_info(T, size). 74 | 75 | d(T, K) -> 76 | mnesia:dirty_delete({T, K}), 77 | mnesia:table_info(T, size). 78 | 79 | do(T, K, V) -> 80 | mnesia:dirty_delete_object(T, {x, K, V}), 81 | mnesia:table_info(T, size). 82 | 83 | await(T) -> 84 | ?m(ok, mnesia:wait_for_tables([T], 10000)). 85 | -------------------------------------------------------------------------------- /test/mnesia_eleveldb_tlib.erl: -------------------------------------------------------------------------------- 1 | %%---------------------------------------------------------------- 2 | %% Copyright (c) 2013-2016 Klarna AB 3 | %% 4 | %% This file is provided to you under the Apache License, 5 | %% Version 2.0 (the "License"); you may not use this file 6 | %% except in compliance with the License. You may obtain 7 | %% a copy of the License at 8 | %% 9 | %% http://www.apache.org/licenses/LICENSE-2.0 10 | %% 11 | %% Unless required by applicable law or agreed to in writing, 12 | %% software distributed under the License is distributed on an 13 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | %% KIND, either express or implied. See the License for the 15 | %% specific language governing permissions and limitations 16 | %% under the License. 17 | %%---------------------------------------------------------------- 18 | 19 | -module(mnesia_eleveldb_tlib). 20 | 21 | -export([start_mnesia/0, 22 | start_mnesia/1, 23 | create_table/1, 24 | create_table/3, 25 | trace/2]). 26 | 27 | 28 | start_mnesia() -> 29 | start_mnesia(false). 30 | 31 | start_mnesia(Mode) -> 32 | if Mode==reset -> 33 | mnesia:delete_schema([node()]), 34 | mnesia:create_schema([node()], 35 | [{backend_types, 36 | [{ldb,mnesia_eleveldb}]}]); 37 | true -> ok 38 | end, 39 | mnesia:start(). 40 | 41 | create_table(Backend) -> 42 | create_table(Backend, [k,v], [v]). 43 | 44 | create_table(Backend, Attrs, Indexes) -> 45 | mnesia:create_table(t, [{index,Indexes}, {attributes,Attrs}, 46 | {Backend, [node()]}]). 47 | 48 | trace(F, Ms) -> 49 | dbg:tracer(), 50 | [tp(M) || M <- Ms], 51 | dbg:p(all,[c]), 52 | try F() 53 | after 54 | [ctp(M) || M <- Ms], 55 | dbg:stop() 56 | end. 57 | 58 | tp({l,M} ) -> dbg:tpl(M,x); 59 | tp({g,M} ) -> dbg:tp(M,x); 60 | tp({l,M,F}) -> dbg:tpl(M,F,x); 61 | tp({g,M,F}) -> dbg:tp(M,F,x). 62 | 63 | ctp({l,M} ) -> dbg:ctpl(M); 64 | ctp({g,M} ) -> dbg:ctp(M); 65 | ctp({l,M,F}) -> dbg:ctpl(M,F); 66 | ctp({g,M,F}) -> dbg:ctp(M,F). 67 | -------------------------------------------------------------------------------- /test/mnesia_eleveldb_xform.erl: -------------------------------------------------------------------------------- 1 | %%---------------------------------------------------------------- 2 | %% Copyright (c) 2013-2016 Klarna AB 3 | %% 4 | %% This file is provided to you under the Apache License, 5 | %% Version 2.0 (the "License"); you may not use this file 6 | %% except in compliance with the License. You may obtain 7 | %% a copy of the License at 8 | %% 9 | %% http://www.apache.org/licenses/LICENSE-2.0 10 | %% 11 | %% Unless required by applicable law or agreed to in writing, 12 | %% software distributed under the License is distributed on an 13 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | %% KIND, either express or implied. See the License for the 15 | %% specific language governing permissions and limitations 16 | %% under the License. 17 | %%---------------------------------------------------------------- 18 | 19 | %% This module is used to test backend plugin extensions to the mnesia 20 | %% backend. It also indirectly tests the mnesia backend plugin 21 | %% extension machinery 22 | %% 23 | %% Usage: mnesia_ext_eleveldb_test:recompile(Extension). 24 | %% Usage: mnesia_ext_eleveldb_test:recompile(). 25 | %% This command is executed in the release/tests/test_server directory 26 | %% before running the normal tests. The command patches the test code, 27 | %% via a parse_transform, to replace disc_only_copies with the Alias. 28 | 29 | -module(mnesia_eleveldb_xform). 30 | 31 | -author("roland.karlsson@erlang-solutions.com"). 32 | -author("ulf.wiger@klarna.com"). 33 | 34 | %% EXPORTS %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 35 | 36 | %% Exporting API 37 | -export([recompile/0, recompile/1]). 38 | 39 | %% Exporting parse_transform callback 40 | -export([parse_transform/2]). 41 | 42 | %% Exporting replacement for mnesia:create_table/2 43 | -export([create_table/1, create_table/2, rpc/4]). 44 | 45 | %% API %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 46 | 47 | %% Recompiling the test code, replacing disc_only_copies with 48 | %% Extension. 49 | recompile() -> 50 | [{Module,Alias}|_] = extensions(), 51 | recompile(Module, Alias). 52 | 53 | recompile(MorA) -> 54 | case { lists:keyfind(MorA, 1, extensions()), 55 | lists:keyfind(MorA, 2, extensions()) 56 | } of 57 | {{Module,Alias}, _} -> 58 | recompile(Module, Alias); 59 | {false, {Module,Alias}} -> 60 | recompile(Module, Alias); 61 | {false,false} -> 62 | {error, cannot_find_module_or_alias} 63 | end. 64 | 65 | recompile(Module, Alias) -> 66 | io:format("recompile(~p,~p)~n",[Module, Alias]), 67 | put_ext(module, Module), 68 | put_ext(alias, Alias), 69 | Modules = [ begin {M,_} = lists:split(length(F)-4, F), 70 | list_to_atom(M) end || 71 | F <- begin {ok,L} = file:list_dir("."), L end, 72 | lists:suffix(".erl", F), 73 | F=/= atom_to_list(?MODULE) ++ ".erl" ], 74 | io:format("Modules = ~p~n",[Modules]), 75 | lists:foreach(fun(M) -> 76 | c:c(M, [{parse_transform, ?MODULE}]) 77 | end, Modules). 78 | 79 | %% TEST REPLACEMENT CALLBACKS %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 80 | 81 | %% replacement for mnesia:create_table that ensures that 82 | create_table(Name, Parameters) -> 83 | create_table([{name,Name} | Parameters]). 84 | 85 | create_table(Parameters) when is_list(Parameters) -> 86 | case lists:keymember(leveldb_copies, 1, Parameters) of 87 | true -> 88 | %% case lists:member({type, bag}, Parameters) of 89 | %% true -> 90 | %% ct:comment("ERROR: Contains leveldb table with bag"), 91 | %% {aborted, {leveldb_does_not_support_bag, Parameters}}; 92 | %% false -> 93 | ct:comment("INFO: Contains leveldb table"), 94 | io:format("INFO: create_table(~p)~n", [Parameters]), 95 | mnesia:create_table(Parameters); 96 | %% end; 97 | false -> 98 | mnesia:create_table(Parameters) 99 | end; 100 | create_table(Param) -> 101 | %% Probably bad input, e.g. from mnesia_evil_coverage_SUITE.erl 102 | mnesia:create_table(Param). 103 | 104 | 105 | rpc(N, mnesia, start, [Opts]) -> 106 | case lists:keymember(schema, 1, Opts) of 107 | true -> rpc:call(N, mnesia, call, [Opts]); 108 | false -> 109 | Opts1 = [{schema, [{backend_types, backends()}]}|Opts], 110 | rpc:call(N, mnesia, start, [Opts1]) 111 | end; 112 | rpc(N, M, F, A) -> 113 | rpc:call(N, M, F, A). 114 | 115 | 116 | %% PARSE_TRANSFORM CALLBACK %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 117 | 118 | %% The callback for c:c(Module, [{parse_transform,?MODULE}]) 119 | parse_transform(Forms, _Options) -> 120 | plain_transform(fun do_transform/1, Forms). 121 | 122 | do_transform({'attribute', _, module, Module}) -> 123 | io:format("~n~nMODULE: ~p~n", [Module]), 124 | continue; 125 | do_transform({'atom', Line, disc_only_copies}) -> 126 | io:format("replacing disc_only_copies with ~p~n", [get_ext(alias)]), 127 | {'atom', Line, get_ext(alias)}; 128 | do_transform(Form = { call, L1, 129 | { remote, L2, 130 | {atom, L3, mnesia}, 131 | {atom, L4, create_table}}, 132 | Arguments}) -> 133 | NewForm = { call, L1, 134 | { remote, L2, 135 | {atom, L3, ?MODULE}, 136 | {atom, L4, create_table}}, 137 | plain_transform(fun do_transform/1, Arguments)}, 138 | io:format("~nConvert Form:~n~s~n~s~n", [pp_form(Form), pp_form(NewForm)]), 139 | NewForm; 140 | do_transform(Form = { call, L1, 141 | { remote, L2, 142 | {atom, L3, rpc}, 143 | {atom, L4, call}}, 144 | [{var, _, _} = N, {atom, _, mnesia} = Mnesia, 145 | {atom, _, start} = Start, Args]}) -> 146 | NewForm = { call, L1, { remote, L2, 147 | {atom, L3, ?MODULE}, 148 | {atom, L4, rpc}}, 149 | [N, Mnesia, Start, Args]}, 150 | io:format("~nConvert Form:~n~s~n~s~n", [pp_form(Form), pp_form(NewForm)]), 151 | NewForm; 152 | 153 | do_transform(Form = { call, L1, 154 | { remote, L2, 155 | {atom, L3, mnesia}, 156 | {atom, L4, create_schema}}, 157 | [Nodes]}) -> 158 | P = element(2, Nodes), 159 | NewForm = { call, L1, 160 | { remote, L2, 161 | {atom, L3, mnesia}, 162 | {atom, L4, create_schema}}, 163 | [Nodes, erl_parse:abstract([{backend_types, backends()}], P)]}, 164 | io:format("~nConvert Form:~n~s~n~s~n", [pp_form(Form), pp_form(NewForm)]), 165 | NewForm; 166 | do_transform(Form = { call, L1, 167 | { remote, L2, 168 | {atom, L3, mnesia}, 169 | {atom, L4, start}}, 170 | []}) -> 171 | NewForm = { call, L1, 172 | { remote, L2, 173 | {atom, L3, mnesia}, 174 | {atom, L4, start}}, 175 | [erl_parse:abstract( 176 | [{schema, [{backend_types, backends()}]}], L4)]}, 177 | io:format("~nConvert Form:~n~s~n~s~n", [pp_form(Form), pp_form(NewForm)]), 178 | NewForm; 179 | do_transform(Form = { call, L1, 180 | { remote, L2, 181 | {atom, L3, mnesia}, 182 | {atom, L4, start}}, 183 | [Opts]}) -> 184 | P = element(2, Opts), 185 | NewForm = { call, L1, 186 | { remote, L2, 187 | {atom, L3, mnesia}, 188 | {atom, L4, start}}, 189 | [{cons, P, 190 | erl_parse:abstract( 191 | {schema, [{backend_types, backends()}]}, L4), Opts}]}, 192 | io:format("~nConvert Form:~n~s~n~s~n", [pp_form(Form), pp_form(NewForm)]), 193 | NewForm; 194 | 195 | %% 1354:unsupp_user_props(doc) -> 196 | %% 1355: ["Simple test of adding user props in a schema_transaction"]; 197 | %% 1356:unsupp_user_props(suite) -> []; 198 | %% 1357:unsupp_user_props(Config) when is_list(Config) -> 199 | do_transform(Form = { function, L1, F, 1, [C1, C2, C3] }) 200 | when F == unsupp_user_props -> 201 | L3 = element(2, C3), 202 | NewForm = { function, L1, F, 1, 203 | [C1, C2, {clause, L3, [{var, L3, '_'}], [], 204 | [{tuple, L3, [{atom, L3, skip}, 205 | erl_parse:abstract( 206 | "Skipped for leveldb test", L3)]} 207 | ]} ] }, 208 | io:format("~nConvert Form:" 209 | "~n=============~n~s" 210 | "==== To: ====~n~s" 211 | "=============~n", 212 | [cut(20, pp_form(Form)), cut(20, pp_form(NewForm))]), 213 | NewForm; 214 | do_transform(Form = { function, L1, F, 1, [C1, C2] }) 215 | when F == storage_options -> 216 | L2 = element(2, C2), 217 | NewForm = { function, L1, F, 1, 218 | [C1, {clause, L2, [{var, L2, '_'}], [], 219 | [{tuple, L2, [{atom, L2, skip}, 220 | erl_parse:abstract( 221 | "Skipped for leveldb test", L2)]} 222 | ]} ] }, 223 | io:format("~nConvert Form:" 224 | "~n=============~n~s" 225 | "==== To: ====~n~s" 226 | "=============~n", 227 | [cut(20, pp_form(Form)), cut(20, pp_form(NewForm))]), 228 | NewForm; 229 | do_transform(_Form) -> 230 | continue. 231 | 232 | pp_form(F) when element(1,F) == attribute; element(1,F) == function -> 233 | erl_pp:form(F); 234 | pp_form(F) -> 235 | erl_pp:expr(F). 236 | 237 | cut(Lines, S) -> 238 | case re:split(S, "\\v", [{return,list}]) of 239 | Lns when length(Lns) =< Lines -> 240 | S; 241 | Lns -> 242 | lists:flatten( 243 | add_lf(lists:sublist(Lns, 1, Lines) ++ ["...\n"])) 244 | end. 245 | 246 | add_lf([H|T]) -> 247 | [H | ["\n" ++ L || L <- T]]. 248 | 249 | %% INTERNAL %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 250 | 251 | %% A trick for doing parse transforms easier 252 | 253 | plain_transform(Fun, Forms) when is_function(Fun, 1), is_list(Forms) -> 254 | plain_transform1(Fun, Forms). 255 | 256 | plain_transform1(_, []) -> 257 | []; 258 | plain_transform1(Fun, [F|Fs]) when is_atom(element(1,F)) -> 259 | case Fun(F) of 260 | continue -> 261 | [list_to_tuple(plain_transform1(Fun, tuple_to_list(F))) | 262 | plain_transform1(Fun, Fs)]; 263 | {done, NewF} -> 264 | [NewF | Fs]; 265 | {error, Reason} -> 266 | io:format("Error: ~p (~p)~n", [F,Reason]); 267 | NewF when is_tuple(NewF) -> 268 | [NewF | plain_transform1(Fun, Fs)] 269 | end; 270 | plain_transform1(Fun, [L|Fs]) when is_list(L) -> 271 | [plain_transform1(Fun, L) | plain_transform1(Fun, Fs)]; 272 | plain_transform1(Fun, [F|Fs]) -> 273 | [F | plain_transform1(Fun, Fs)]; 274 | plain_transform1(_, F) -> 275 | F. 276 | 277 | %% Existing extensions. 278 | %% NOTE: The first is default. 279 | extensions() -> 280 | [ {mnesia_eleveldb, leveldb_copies} 281 | ]. 282 | %% {mnesia_ext_filesystem, fs_copies}, 283 | %% {mnesia_ext_filesystem, fstab_copies}, 284 | %% {mnesia_ext_filesystem, raw_fs_copies} 285 | %% ]. 286 | 287 | backends() -> 288 | [{T,M} || {M,T} <- extensions()]. 289 | 290 | %% Process global storage 291 | 292 | put_ext(Key, Value) -> 293 | ets:insert(global_storage(), {Key, Value}). 294 | 295 | global_storage() -> 296 | case whereis(?MODULE) of 297 | undefined -> 298 | Me = self(), 299 | P = spawn(fun() -> 300 | T = ets:new(?MODULE, [public,named_table]), 301 | init_ext(T), 302 | register(?MODULE, self()), 303 | Me ! {self(), done}, 304 | wait() 305 | end), 306 | receive {P, done} -> 307 | ok 308 | end; 309 | _ -> 310 | ok 311 | end, 312 | ?MODULE. 313 | 314 | init_ext(T) -> 315 | [{Mod,Alias}|_] = extensions(), 316 | ets:insert(T, {alias, Alias}), 317 | ets:insert(T, {module, Mod}). 318 | 319 | wait() -> 320 | receive stop -> 321 | ok 322 | end. 323 | 324 | get_ext(Key) -> 325 | case catch ets:lookup(global_storage(), Key) of 326 | [] -> 327 | io:format("Data for ~p not stored~n", [Key]), 328 | undefined; 329 | {'EXIT', Reason} -> 330 | io:format("Get value for ~p failed (~p)~n", [Key, Reason]), 331 | undefined; 332 | [{Key,Value}] -> 333 | Value 334 | end. 335 | --------------------------------------------------------------------------------