├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── priv └── pgsql.tgz ├── src ├── mnesia_pg.app.src ├── mnesia_pg.erl ├── mnesia_pg_app.erl ├── mnesia_pg_conns.erl ├── mnesia_pg_int.hrl ├── mnesia_pg_sup.erl ├── mnesia_pg_util.erl └── mnesia_pgsql_mon.erl ├── stdapp.mk └── test ├── pg_perf.erl ├── pg_proper_semantics.erl └── pg_test.erl /.gitignore: -------------------------------------------------------------------------------- 1 | pgsql_src 2 | pgsql 3 | ebin 4 | *~ 5 | MnesiaCore* 6 | Mnesia.* -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | include ./stdapp.mk 2 | 3 | PSQL_TARGET=$(PWD)/pgsql 4 | APPLICATION_NAME_MACRO=APPNAME 5 | 6 | .PHONY: build-pgsql clean-pgsql 7 | build: build-pgsql 8 | clean: clean-pgsql 9 | 10 | build-pgsql: pgsql $(ERL_OBJECTS) $(APP_FILE) 11 | @$(ERL_NOSHELL) -eval 'erlang:halt(case file:consult("$(APP_FILE)") of {ok,_}->0; _->1 end)' || { echo '*** error: $(APP_FILE) is not readable'; exit 1; } 12 | 13 | pgsql_src/src: 14 | mkdir -p pgsql_src 15 | (cd pgsql_src && tar xzf ../priv/pgsql.tgz) 16 | 17 | pgsql: pgsql_src/src 18 | mkdir -p $(PSQL_TARGET) 19 | (cd pgsql_src && \ 20 | ./configure --prefix=$(PSQL_TARGET) \ 21 | && make && make install) 22 | 23 | clean-pgsql: 24 | rm -f $(ERL_DEPS) $(ERL_TEST_DEPS) 25 | rm -rf pgsql 26 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # mnesia_pg 2 | Postgres backend to Mnesia via mnesia_ext 3 | 4 | This is a very raw implementation, made as proof of concept and for preliminary benchmarking. It has not been used in production. Feel free to improve it. 5 | -------------------------------------------------------------------------------- /priv/pgsql.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klarna/mnesia_pg/c4b3fce570e06630d2984b6c61b2e439726259f8/priv/pgsql.tgz -------------------------------------------------------------------------------- /src/mnesia_pg.app.src: -------------------------------------------------------------------------------- 1 | {application, mnesia_pg, 2 | [ 3 | {description, "PG plugin for mnesia"}, 4 | {vsn, "1.0"}, 5 | {registered, []}, 6 | {applications, [ 7 | kernel, 8 | stdlib 9 | ]}, 10 | {mod, {mnesia_pg_app, []}}, 11 | {start_phases, [{check_schema_cookie,[]}]}, 12 | {env, [{pool_size, 3}, {pg_user, "mnesia"}, {pg_pwd, "mnesia"}]} 13 | ]}. 14 | -------------------------------------------------------------------------------- /src/mnesia_pg.erl: -------------------------------------------------------------------------------- 1 | %%% 2 | %%% Copyright (c) 2014-2015 Klarna AB 3 | %%% 4 | %%% This file is provided to you under the Apache License, 5 | %%% Version 2.0 (the "License"); you may not use this file 6 | %%% except in compliance with the License. You may obtain 7 | %%% a copy of the License at 8 | %%% 9 | %%% http://www.apache.org/licenses/LICENSE-2.0 10 | %%% 11 | %%% Unless required by applicable law or agreed to in writing, 12 | %%% software distributed under the License is distributed on an 13 | %%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | %%% KIND, either express or implied. See the License for the 15 | %%% specific language governing permissions and limitations 16 | %%% under the License. 17 | 18 | %% TODO: 19 | %% io wait is high 20 | %% index is scanned? 21 | 22 | %%%header_doc_include 23 | 24 | -module(mnesia_pg). 25 | 26 | %% ---------------------------------------------------------------------------- 27 | %% BEHAVIOURS 28 | %% ---------------------------------------------------------------------------- 29 | 30 | -behaviour(mnesia_backend_type). 31 | 32 | %% ---------------------------------------------------------------------------- 33 | %% EXPORTS 34 | %% ---------------------------------------------------------------------------- 35 | 36 | %% 37 | %% CONVENIENCE API 38 | %% 39 | 40 | -export([register/0, 41 | dbg/1, 42 | default_alias/0]). 43 | 44 | %% 45 | %% DEBUG API 46 | %% 47 | 48 | -export([show_table/1, 49 | show_table/2, 50 | show_table/3, 51 | fold/6]). 52 | 53 | %% 54 | %% BACKEND CALLBACKS 55 | %% 56 | 57 | %% backend management 58 | -export([init_backend/0, 59 | add_aliases/1, 60 | remove_aliases/1]). 61 | 62 | %% schema level callbacks 63 | -export([semantics/2, 64 | check_definition/4, 65 | create_table/3, 66 | load_table/4, 67 | close_table/2, 68 | sync_close_table/2, 69 | delete_table/2, 70 | info/3]). 71 | 72 | %% table synch calls 73 | -export([sender_init/4, 74 | sender_handle_info/5, 75 | receiver_first_message/4, 76 | receive_data/5, 77 | receive_done/4]). 78 | 79 | %% low-level accessor callbacks. 80 | -export([delete/3, 81 | first/2, 82 | fixtable/3, 83 | insert/3, 84 | last/2, 85 | lookup/3, 86 | match_delete/3, 87 | next/3, 88 | prev/3, 89 | repair_continuation/2, 90 | select/1, 91 | select/3, 92 | select/4, 93 | slot/3, 94 | update_counter/4]). 95 | 96 | %% Index consistency 97 | -export([index_is_consistent/3, 98 | is_index_consistent/2]). 99 | 100 | %% record and key validation 101 | -export([validate_key/6, 102 | validate_record/6]). 103 | 104 | %% file extension callbacks 105 | -export([real_suffixes/0, 106 | tmp_suffixes/0]). 107 | 108 | -export([ix_prefixes/3]). 109 | 110 | %%%header_doc_include 111 | 112 | %%%impl_doc_include 113 | 114 | %% ---------------------------------------------------------------------------- 115 | %% INCLUDES 116 | %% ---------------------------------------------------------------------------- 117 | 118 | -include_lib("mnesia/src/mnesia.hrl"). 119 | -include_lib("kernel/include/file.hrl"). 120 | 121 | %% ---------------------------------------------------------------------------- 122 | %% DEFINES 123 | %% ---------------------------------------------------------------------------- 124 | 125 | %% Data and meta data (a.k.a. info) are stored in the same table. 126 | %% This is a table of the first byte in data 127 | %% 0 = before meta data 128 | %% 1 = meta data 129 | %% 2 = before data 130 | %% >= 8 = data 131 | 132 | -ifdef(DEBUG). 133 | -define(dbg(E), E). 134 | -else. 135 | -define(dbg(E), ok). 136 | -endif. 137 | %% ---------------------------------------------------------------------------- 138 | %% RECORDS 139 | %% ---------------------------------------------------------------------------- 140 | 141 | -record(sel, {alias, % TODO: not used 142 | tab, 143 | ref, 144 | keypat, 145 | limit}). 146 | 147 | %% ---------------------------------------------------------------------------- 148 | %% CONVENIENCE API 149 | %% ---------------------------------------------------------------------------- 150 | 151 | register() -> 152 | Alias = default_alias(), 153 | Module = ?MODULE, 154 | mnesia:add_backend_type(Alias, Module), 155 | {ok, {Alias, Module}}. 156 | 157 | default_alias() -> 158 | pg_copies. 159 | 160 | 161 | %% ---------------------------------------------------------------------------- 162 | %% DEBUG API 163 | %% ---------------------------------------------------------------------------- 164 | 165 | %% A debug function that shows the table content 166 | show_table(Tab) -> 167 | show_table(default_alias(), Tab). 168 | 169 | show_table(Alias, Tab) -> 170 | show_table(Alias, Tab, 100). 171 | 172 | show_table(Alias, Tab, Limit) -> 173 | {C, _} = Ref = get_ref(Alias, Tab), 174 | try 175 | with_iterator(Ref, fun(Curs) -> i_show_table(Curs, Limit) end) 176 | after 177 | mnesia_pg_conns:free(C) 178 | end. 179 | 180 | %% PRIVATE 181 | 182 | i_show_table(_, 0) -> 183 | {error, skipped_some}; 184 | i_show_table(Curs, Limit) -> 185 | case fetch_next(Curs) of 186 | {ok, EncKey, EncVal} -> 187 | K = decode_key(EncKey), 188 | V = decode_val(EncVal), 189 | io:fwrite("~p: ~p~n", [K, V]), 190 | i_show_table(Curs, Limit-1); 191 | _ -> 192 | ok 193 | end. 194 | 195 | %% ---------------------------------------------------------------------------- 196 | %% BACKEND CALLBACKS 197 | %% ---------------------------------------------------------------------------- 198 | 199 | %% backend management 200 | 201 | init_backend() -> 202 | application:start(mnesia_pg). 203 | 204 | add_aliases(_Aliases) -> 205 | ok. 206 | 207 | remove_aliases(_Aliases) -> 208 | ok. 209 | 210 | %% schema level callbacks 211 | 212 | %% This function is used to determine what the plugin supports 213 | %% semantics(Alias, storage) -> 214 | %% ram_copies | disc_copies | disc_only_copies (mandatory) 215 | %% semantics(Alias, types) -> 216 | %% [bag | set | ordered_set] (mandatory) 217 | %% semantics(Alias, index_fun) -> 218 | %% fun(Alias, Tab, Pos, Obj) -> [IxValue] (optional) 219 | %% semantics(Alias, _) -> 220 | %% undefined. 221 | %% 222 | semantics(_Alias, storage) -> disc_only_copies; 223 | semantics(_Alias, types ) -> [set, ordered_set]; 224 | semantics(_Alias, index_types) -> [ordered]; 225 | semantics(_Alias, index_fun) -> fun index_f/4; 226 | semantics(_Alias, _) -> undefined. 227 | 228 | is_index_consistent(_Alias, _) -> 229 | true. 230 | 231 | index_is_consistent(_Alias, _, __Bool) -> 232 | ok. 233 | 234 | %% PRIVATE FUN 235 | index_f(_Alias, _Tab, Pos, Obj) -> 236 | [element(Pos, Obj)]. 237 | 238 | ix_prefixes(_Tab, _Pos, Obj) -> 239 | lists:foldl( 240 | fun(V, Acc) when is_list(V) -> 241 | try Pfxs = prefixes(list_to_binary(V)), 242 | Pfxs ++ Acc 243 | catch 244 | error:_ -> 245 | Acc 246 | end; 247 | (V, Acc) when is_binary(V) -> 248 | Pfxs = prefixes(V), 249 | Pfxs ++ Acc; 250 | (_, Acc) -> 251 | Acc 252 | end, [], tl(tuple_to_list(Obj))). 253 | 254 | prefixes(<>) -> 255 | [P]; 256 | prefixes(_) -> 257 | []. 258 | 259 | %% For now, only verify that the type is set or ordered_set. 260 | %% set is OK as ordered_set is a kind of set. 261 | check_definition(Alias, Tab, Nodes, Props) -> 262 | Id = {Alias, Nodes}, 263 | Props1 = lists:map( 264 | fun({type, T} = P) -> 265 | if T==set; T==ordered_set -> 266 | P; 267 | true -> 268 | mnesia:abort({combine_error, 269 | Tab, 270 | [Id, {type,T}]}) 271 | end; 272 | ({user_properties, _} = P) -> 273 | P; 274 | (P) -> P 275 | end, Props), 276 | {ok, Props1}. 277 | 278 | -ifdef(DEBUG). 279 | pp_calls(I, [{M,F,A,Pos} | T]) -> 280 | Spc = lists:duplicate(I, $\s), 281 | Pp = fun(Mx,Fx,Ax,Px) -> 282 | [atom_to_list(Mx),":",atom_to_list(Fx),"/",integer_to_list(Ax), 283 | pp_pos(Px)] 284 | end, 285 | [Pp(M,F,A,Pos)|[["\n",Spc,Pp(M1,F1,A1,P1)] || {M1,F1,A1,P1} <- T]]. 286 | pp_pos([]) -> ""; 287 | pp_pos(L) when is_integer(L) -> 288 | [" (", integer_to_list(L), ")"]; 289 | pp_pos([{file,_},{line,L}]) -> 290 | [" (", integer_to_list(L), ")"]. 291 | 292 | dbg_caller() -> 293 | try 1=2 294 | catch 295 | error:_ -> 296 | tl(erlang:get_stacktrace()) 297 | end. 298 | -endif. 299 | 300 | sync_close_table(_Alias, _Tab) -> 301 | ok. 302 | 303 | info(Alias, Tab, size) -> 304 | table_count(Alias, Tab); % not fast... 305 | info(Alias, Tab, memory) -> 306 | table_size(Alias, Tab); 307 | info(_Alias, _Tab, _) -> 308 | undefined. 309 | 310 | %% =========================================================== 311 | %% Table synch protocol 312 | %% Callbacks are 313 | %% Sender side: 314 | %% 1. sender_init(Alias, Tab, RemoteStorage, ReceiverPid) -> 315 | %% {standard, InitFun, ChunkFun} | {InitFun, ChunkFun} when 316 | %% InitFun :: fun() -> {Recs, Cont} | '$end_of_table' 317 | %% ChunkFun :: fun(Cont) -> {Recs, Cont1} | '$end_of_table' 318 | %% 319 | %% If {standard, I, C} is returned, the standard init message will be 320 | %% sent to the receiver. Matching on RemoteStorage can reveal if a 321 | %% different protocol can be used. 322 | %% 323 | %% 2. InitFun() is called 324 | %% 3a. ChunkFun(Cont) is called repeatedly until done 325 | %% 3b. sender_handle_info(Msg, Alias, Tab, ReceiverPid, Cont) -> 326 | %% {ChunkFun, NewCont} 327 | %% 328 | %% Receiver side: 329 | %% 1. receiver_first_message(SenderPid, Msg, Alias, Tab) -> 330 | %% {Size::integer(), State} 331 | %% 2. receive_data(Data, Alias, Tab, _Sender, State) -> 332 | %% {more, NewState} | {{more, Msg}, NewState} 333 | %% 3. receive_done(_Alias, _Tab, _Sender, _State) -> 334 | %% ok 335 | %% 336 | %% The receiver can communicate with the Sender by returning 337 | %% {{more, Msg}, St} from receive_data/4. The sender will be called through 338 | %% sender_handle_info(Msg, ...), where it can adjust its ChunkFun and 339 | %% Continuation. Note that the message from the receiver is sent once the 340 | %% receive_data/4 function returns. This is slightly different from the 341 | %% normal mnesia table synch, where the receiver acks immediately upon 342 | %% reception of a new chunk, then processes the data. 343 | %% 344 | 345 | sender_init(Alias, Tab, _RemoteStorage, _Pid) -> 346 | %% Need to send a message to the receiver. It will be handled in 347 | %% receiver_first_message/4 below. There could be a volley of messages... 348 | {standard, 349 | fun() -> 350 | select(Alias, Tab, [{'_',[],['$_']}], 100) 351 | end, 352 | chunk_fun()}. 353 | 354 | sender_handle_info(_Msg, _Alias, _Tab, _ReceiverPid, Cont) -> 355 | %% ignore - we don't expect any message from the receiver 356 | {chunk_fun(), Cont}. 357 | 358 | receiver_first_message(_Pid, {first, Size} = _Msg, _Alias, _Tab) -> 359 | {Size, _State = []}. 360 | 361 | receive_data(Data, Alias, Tab, _Sender, State) -> 362 | [insert(Alias, Tab, Obj) || Obj <- Data], 363 | {more, State}. 364 | 365 | receive_done(_Alias, _Tab, _Sender, _State) -> 366 | ok. 367 | 368 | chunk_fun() -> 369 | fun(Cont) -> 370 | select(Cont) 371 | end. 372 | 373 | %% End of table synch protocol 374 | %% =========================================================== 375 | 376 | delete(Alias, Tab, Key) -> 377 | delete_from(Alias, Tab, encode_key(Key)). 378 | 379 | %% Not relevant for an ordered_set 380 | fixtable(_Alias, _Tab, _Bool) -> 381 | true. 382 | 383 | insert(Alias, Tab, Obj) -> 384 | Pos = keypos(Tab), 385 | Key = element(Pos, Obj), 386 | PKey = encode_key(Key), 387 | SKey = encode_lookup_key(Key), 388 | Val = encode_val(Obj), 389 | upsert_into(Alias, Tab, PKey, SKey, Val). 390 | 391 | find_row(_, _, []) -> 392 | []; 393 | find_row(_, _, [Row]) -> 394 | [decode_val(element(1,Row))]; 395 | find_row(Key, Pos, [Row|Rs]) -> 396 | Rec = decode_val(element(1,Row)), 397 | case element(Pos, Rec) of 398 | Key -> 399 | [Rec]; 400 | _ -> 401 | find_row(Key, Pos, Rs) 402 | end. 403 | 404 | lookup(Alias, Tab0, Key) -> 405 | {C, Tab} = get_ref(Alias, Tab0), 406 | SKey = encode_lookup_key(Key), 407 | try 408 | {ok, _, Rs} = pgsql:equery(C, ["select erlval from ", Tab, " where erlhash=$1"], [SKey]), 409 | find_row(Key, keypos(Tab0), Rs) 410 | after 411 | mnesia_pg_conns:free(C) 412 | end. 413 | 414 | match_delete(Alias, Tab, Pat) when is_atom(Pat) -> 415 | case is_wild(Pat) of 416 | true -> 417 | delete_all(Alias, Tab), 418 | ok; 419 | false -> 420 | %% can this happen?? 421 | error(badarg) 422 | end; 423 | match_delete(Alias, Tab, Pat) when is_tuple(Pat) -> 424 | KP = keypos(Tab), 425 | Key = element(KP, Pat), 426 | case is_wild(Key) of 427 | true -> 428 | delete_all(Alias, Tab), 429 | ok; 430 | false -> 431 | delete_from_pattern(Alias, Tab, Pat) 432 | end, 433 | ok. 434 | 435 | %% It is tempting to use a cursor for next/prev, however, the ets semantics, used by mnesia, 436 | %% imply that any call to next (prev) may position itself right after (before) the given key, 437 | %% which may not always be the next row in logical order. As there is no open call, 438 | %% it is not clear how to open a cursor that can be leveraged to efficiently find 439 | %% a proper segment to iterate through. 440 | %% Note: prev for a set in mnesia/ets equals next (sic!) 441 | next(Alias, Tab0, Key) -> 442 | {C, Tab} = get_ref(Alias, Tab0), 443 | try 444 | {ok, _, Res} = pgsql:equery(C, ["select erlkey from ", Tab, " where erlkey > $1 order by erlkey limit 1"], [encode_key(Key)]), 445 | case (Res) of 446 | [] -> 447 | '$end_of_table'; 448 | [{TKey}] -> %check pattern 449 | decode_key(TKey) 450 | end 451 | after 452 | mnesia_pg_conns:free(C) 453 | end. 454 | 455 | prev(Alias, Tab0, Key) -> 456 | {C, Tab} = get_ref(Alias, Tab0), 457 | try 458 | {ok, _, Res} = pgsql:equery(C, ["select erlkey from ", Tab, " where erlkey < $1 order by erlkey desc limit 1"], [encode_key(Key)]), 459 | case (Res) of 460 | [] -> 461 | '$end_of_table'; 462 | [{TKey}] -> 463 | decode_key(TKey) 464 | end 465 | after 466 | mnesia_pg_conns:free(C) 467 | end. 468 | 469 | first(Alias, Tab0) -> 470 | {C, Tab} = get_ref(Alias, Tab0), 471 | try 472 | {ok, _, Res} = pgsql:equery(C, ["select erlkey from ", Tab, " order by erlkey limit 1"], []), 473 | case (Res) of 474 | [] -> 475 | '$end_of_table'; 476 | [{TKey}] -> 477 | decode_key(TKey) 478 | end 479 | after 480 | mnesia_pg_conns:free(C) 481 | end. 482 | 483 | last(Alias, Tab0) -> 484 | {C, Tab} = get_ref(Alias, Tab0), 485 | try 486 | {ok, _, Res} = pgsql:equery(C, ["select erlkey from ", Tab, " order by erlkey desc limit 1"], []), 487 | case (Res) of 488 | [] -> 489 | '$end_of_table'; 490 | [{TKey}] -> 491 | decode_key(TKey) 492 | end 493 | after 494 | mnesia_pg_conns:free(C) 495 | end. 496 | 497 | prefix_get({C, Tab}, IsPfx, Pfx) -> 498 | {Where,Vs} = case IsPfx of 499 | true -> {"erlkey >= $1", [Pfx]}; 500 | false -> {"erlkey = $1", [Pfx]} 501 | end, 502 | {ok, _, Res} = 503 | pgsql:equery(C, ["select erlkey, erlval from ", Tab, " where ", 504 | Where, " order by erlkey limit 2"], Vs), 505 | Res. 506 | 507 | prefix_next({C, Tab}, IsPfx, Pfx) -> 508 | {Where,Vs} = case IsPfx of 509 | true -> {"erlkey >= $1", [Pfx]}; 510 | false -> {"erlkey > $1", [Pfx]} 511 | end, 512 | {ok, _, Res} = 513 | pgsql:equery(C, ["select erlkey, erlval from ", Tab, " where ", 514 | Where, " order by erlkey limit 2"], Vs), 515 | Res. 516 | 517 | repair_continuation(Cont, _Ms) -> 518 | Cont. 519 | 520 | select(C) -> 521 | Cont = get_sel_cont(C), 522 | Cont(). 523 | 524 | select(Alias, Tab, Ms) -> 525 | case select(Alias, Tab, Ms, infinity) of 526 | {Res, '$end_of_table'} -> 527 | Res; 528 | '$end_of_table' -> 529 | '$end_of_table' 530 | end. 531 | 532 | select(Alias, Tab0, Ms, Limit) when Limit==infinity; is_integer(Limit) -> 533 | {C, _} = Ref = get_ref(Alias, Tab0), 534 | try 535 | do_select(Ref, Tab0, Ms, Limit) 536 | after 537 | mnesia_pg_conns:free(C) 538 | end. 539 | 540 | slot(Alias, Tab0, Pos) when is_integer(Pos), Pos >= 0 -> 541 | {C, _} = Ref = get_ref(Alias, Tab0), 542 | F = fun(Curs) -> slot_iter_set(fetch_next(Curs), Curs, 0, Pos) end, 543 | try 544 | with_iterator(Ref, F) 545 | after 546 | mnesia_pg_conns:free(C) 547 | end; 548 | slot(_, _, _) -> 549 | error(badarg). 550 | 551 | 552 | %% Exactly which objects Mod:slot/2 is supposed to return is not defined, 553 | %% so let's just use the same version for both set and bag. No one should 554 | %% use this function anyway, as it is ridiculously inefficient. 555 | slot_iter_set({ok, _K, V}, _Curs, P, P) -> 556 | [decode_val(V)]; 557 | slot_iter_set({ok, _, _}, Curs, P1, P) when P1 < P -> 558 | slot_iter_set(fetch_next(Curs), Curs, P1+1, P); 559 | slot_iter_set(Res, _, _, _) when element(1, Res) =/= ok -> 560 | '$end_of_table'. 561 | 562 | %% update value of key by incrementing 563 | %% need select for update since the value in erlval is an int encoded as a binary 564 | update_counter(Alias, Tab0, Key, Val) when is_integer(Val) -> 565 | {C, Tab} = get_ref(Alias, Tab0), 566 | PKey = encode_key(Key), 567 | try 568 | {ok, [], []} = pgsql:squery(C, "begin"), 569 | {ok, _, Res} = pgsql:equery(C, ["select erlval from ", Tab, " where erlkey=$1 for update"], [PKey]), 570 | case (length(Res)) of 571 | 0 -> 572 | pgsql:squery(C, "commit"), 573 | badarg; 574 | _ -> 575 | [Row] = Res, 576 | Rec = decode_val(element(1, Row)), 577 | case Rec of 578 | {T,K,Old} when is_integer(Old) -> 579 | New = Old+Val, 580 | {ok, _} = pgsql:equery(C, 581 | ["update ", Tab, " set erlval=$2, change_time=current_timestamp where erlkey=$1"], 582 | [PKey, encode_val({T,K,New})] 583 | ), 584 | pgsql:squery(C, "commit"); 585 | _ -> 586 | pgsql:squery(C, "rollback"), 587 | New = badarg 588 | end, 589 | New 590 | end 591 | after 592 | mnesia_pg_conns:free(C) 593 | end. 594 | 595 | %% PRIVATE 596 | 597 | with_iterator({C, Tab}, F) -> 598 | Curs = open_cursor(C, Tab), 599 | try 600 | R = F(Curs), 601 | commit_cursor(Curs), 602 | R 603 | catch 604 | error:Reason -> 605 | io:fwrite("Cursor ~p failed (~p): ~p~n", [Curs, Reason, erlang:get_stacktrace()]), 606 | rollback_cursor(Curs), 607 | '$end_of_table' 608 | end. 609 | 610 | with_positioned_iterator({C, Tab}, Pat, F) -> 611 | Curs = open_cursor(C, Tab, Pat), 612 | try 613 | R = F(Curs), 614 | commit_cursor(Curs), 615 | R 616 | catch 617 | error:Reason -> 618 | io:fwrite("Cursor ~p failed (~p): ~p~n", [Curs, Reason, erlang:get_stacktrace()]), 619 | rollback_cursor(Curs), 620 | '$end_of_table' 621 | end. 622 | 623 | %% record and key validation 624 | 625 | validate_key(_Alias, _Tab, RecName, Arity, Type, _Key) -> 626 | {RecName, Arity, Type}. 627 | 628 | validate_record(_Alias, _Tab, RecName, Arity, Type, _Obj) -> 629 | {RecName, Arity, Type}. 630 | 631 | %% file extension callbacks 632 | 633 | %% Extensions for files that are permanent. Needs to be cleaned up 634 | %% e.g. at deleting the schema. 635 | real_suffixes() -> 636 | []. 637 | 638 | %% Extensions for temporary files. Can be cleaned up when mnesia 639 | %% cleans up other temporary files. 640 | tmp_suffixes() -> 641 | []. 642 | 643 | %% ---------------------------------------------------------------------------- 644 | %% PRIVATE SELECT MACHINERY 645 | %% ---------------------------------------------------------------------------- 646 | 647 | do_select(Ref, Tab, MS, Limit) -> 648 | do_select(Ref, Tab, MS, false, Limit). 649 | 650 | do_select(Ref, Tab, MS, AccKeys, Limit) when is_boolean(AccKeys) -> 651 | Keypat = keypat(MS, keypos(Tab)), 652 | Sel = #sel{ref = Ref, 653 | keypat = Keypat, 654 | limit = Limit}, 655 | {IsPfx, Pfx, _} = Keypat, 656 | CompMS = ets:match_spec_compile(MS), 657 | select_traverse(prefix_get(Ref, IsPfx, Pfx), Ref, Limit, Pfx, IsPfx, Pfx, 658 | CompMS, Sel, AccKeys, []). 659 | %% case (Pfx) of 660 | %% <<>> -> 661 | %% with_iterator(Ref, 662 | %% fun(Curs) -> 663 | %% select_traverse(fetch_next_n(Curs), Curs, Limit, Pfx, CompMS, Sel, AccKeys, []) 664 | %% end); 665 | %% _ -> 666 | %% with_positioned_iterator(Ref, Pfx, 667 | %% fun(Curs) -> 668 | %% select_traverse(fetch_next_n(Curs), Curs, Limit, Pfx, CompMS, Sel, AccKeys, []) 669 | %% end) 670 | %% end. 671 | 672 | extract_vars([H|T]) -> 673 | extract_vars(H) ++ extract_vars(T); 674 | extract_vars(T) when is_tuple(T) -> 675 | extract_vars(tuple_to_list(T)); 676 | extract_vars(T) when T=='$$'; T=='$_' -> 677 | [T]; 678 | extract_vars(T) when is_atom(T) -> 679 | case is_wild(T) of 680 | true -> 681 | [T]; 682 | false -> 683 | [] 684 | end; 685 | extract_vars(_) -> 686 | []. 687 | 688 | intersection(A,B) when is_list(A), is_list(B) -> 689 | A -- (A -- B). 690 | 691 | select_traverse([_|_] = L, Curs, Limit, Pfx, IsPfx, Prev, MS, Sel, AccKeys, Acc) -> 692 | select_traverse_(L, Curs, Limit, Pfx, IsPfx, Prev, MS, Sel, AccKeys, Acc); 693 | select_traverse([], _, _, _, _, _, _, _, _, Acc) -> 694 | {lists:reverse(Acc), '$end_of_table'}. 695 | 696 | select_traverse_([{K,V}|T], Curs, Limit, Pfx, _, Prev, MS, Sel, AccKeys, Acc) -> 697 | case is_prefix(Pfx, K) of 698 | true -> 699 | Rec = decode_val(V), 700 | case ets:match_spec_run([Rec], MS) of 701 | [] -> 702 | select_traverse_(T, Curs, Limit, Pfx, false, K, 703 | MS, Sel, AccKeys, Acc); 704 | [Match] -> 705 | Fun = fun(NewLimit, NewAcc) -> 706 | select_traverse_( 707 | T, Curs, NewLimit, Pfx, false, K, 708 | MS, Sel, AccKeys, NewAcc) 709 | end, 710 | Acc1 = if AccKeys -> 711 | [{K, Match}|Acc]; 712 | true -> 713 | [Match|Acc] 714 | end, 715 | traverse_continue(decr(Limit), Fun, Acc1, Sel) 716 | end; 717 | false -> 718 | {lists:reverse(Acc), '$end_of_table'} 719 | end; 720 | select_traverse_([], Curs, Limit, Pfx, IsPfx, Prev, MS, Sel, AccKeys, Acc) -> 721 | select_traverse(prefix_next(Curs, IsPfx, Prev), 722 | Curs, Limit, Pfx, IsPfx, Prev, MS, Sel, AccKeys, Acc). 723 | 724 | 725 | 726 | is_prefix(A, B) when is_binary(A), is_binary(B) -> 727 | Sa = byte_size(A), 728 | case B of 729 | <> -> 730 | true; 731 | _ -> 732 | false 733 | end. 734 | 735 | decr(I) when is_integer(I) -> 736 | I-1; 737 | decr(infinity) -> 738 | infinity. 739 | 740 | traverse_continue(0, F, Acc, #sel{limit = Limit, ref = Ref}) -> 741 | {lists:reverse(Acc), 742 | fun() -> 743 | with_iterator(Ref, 744 | fun(_) -> 745 | F(Limit, []) 746 | end) 747 | end}; 748 | traverse_continue(Limit, F, Acc, _) -> 749 | F(Limit, Acc). 750 | 751 | keypat([{HeadPat,Gs,_}|_], KeyPos) when is_tuple(HeadPat) -> 752 | KP = element(KeyPos, HeadPat), 753 | KeyVars = extract_vars(KP), 754 | Guards = relevant_guards(Gs, KeyVars), 755 | {IsPfx,Pfx} = mnesia_sext:enc_prefix_sb32(KP), 756 | {IsPfx, Pfx, [{KP, Guards, [true]}]}; 757 | keypat(_, _) -> 758 | {true, <<>>, [{'_',[],[true]}]}. 759 | 760 | relevant_guards(Gs, Vars) -> 761 | case Vars -- ['_'] of 762 | [] -> 763 | []; 764 | Vars1 -> 765 | Fun = 766 | fun(G) -> 767 | Vg = extract_vars(G), 768 | intersection(Vg, Vars1) =/= [] andalso (Vg--Vars1) == [] 769 | end, 770 | lists:filter(Fun, Gs) 771 | end. 772 | 773 | get_sel_cont(C) -> 774 | Cont = case C of 775 | {?MODULE, C1} -> C1; 776 | _ -> C 777 | end, 778 | case Cont of 779 | _ when is_function(Cont, 0) -> 780 | case erlang:fun_info(Cont, module) of 781 | {_, ?MODULE} -> 782 | Cont; 783 | _ -> 784 | erlang:error(badarg) 785 | end; 786 | '$end_of_table' -> 787 | fun() -> '$end_of_table' end; 788 | _ -> 789 | erlang:error(badarg) 790 | end. 791 | 792 | 793 | %% ---------------------------------------------------------------------------- 794 | %% COMMON PRIVATE 795 | %% ---------------------------------------------------------------------------- 796 | 797 | %% Note that since a callback can be used as an indexing backend, we 798 | %% cannot assume that keypos will always be 2. For indexes, the tab 799 | %% name will be {Tab, index, Pos}, and The object structure will be 800 | %% {{IxKey,Key}} for an ordered_set index, and {IxKey,Key} for a bag 801 | %% index. 802 | %% 803 | keypos({_, index, _}) -> 804 | 1; 805 | keypos({_, retainer, _}) -> 806 | 2; 807 | keypos(Tab) when is_atom(Tab) -> 808 | 2. 809 | 810 | encode_lookup_key(Key) -> 811 | erlang:phash2(Key). % not necessarily unique, but good for lookups 812 | 813 | encode_key_prefix(Key) -> 814 | mnesia_sext:enc_prefix_sb32(Key). 815 | 816 | encode_key(Key) -> 817 | mnesia_sext:encode_sb32(Key). 818 | 819 | decode_key(CodedKey) -> 820 | mnesia_sext:decode_sb32(CodedKey). 821 | 822 | encode_val(Val) -> 823 | term_to_binary(Val). 824 | %term_to_binary(Val,[{compressed,5},{minor_version, 1}]). 825 | 826 | decode_val(CodedVal) -> 827 | binary_to_term(CodedVal). 828 | 829 | fold(Alias, Tab0, Fun, Acc, MS, N) -> 830 | {C, _} = Ref = get_ref(Alias, Tab0), 831 | try 832 | do_fold(Ref, Tab0, Fun, Acc, MS, N) 833 | after 834 | mnesia_pg_conns:free(C) 835 | end. 836 | 837 | do_fold(Ref, Tab, Fun, Acc, MS, N) -> 838 | {AccKeys, F} = 839 | if is_function(Fun, 3) -> 840 | {true, fun({K,Obj}, Acc1) -> 841 | Fun(Obj, K, Acc1) 842 | end}; 843 | is_function(Fun, 2) -> 844 | {false, Fun} 845 | end, 846 | do_fold1(do_select(Ref, Tab, MS, AccKeys, N), F, Acc). 847 | 848 | do_fold1('$end_of_table', _, Acc) -> 849 | Acc; 850 | do_fold1({L, Cont}, Fun, Acc) -> 851 | Acc1 = lists:foldl(Fun, Acc, L), 852 | do_fold1(select(Cont), Fun, Acc1). 853 | 854 | is_wild('_') -> 855 | true; 856 | is_wild(A) when is_atom(A) -> 857 | case atom_to_list(A) of 858 | "\$" ++ S -> 859 | try begin 860 | _ = list_to_integer(S), 861 | true 862 | end 863 | catch 864 | error:_ -> 865 | false 866 | end; 867 | _ -> 868 | false 869 | end; 870 | is_wild(_) -> 871 | false. 872 | 873 | 874 | %% PG interface 875 | %% Each table has 3 columns: encoded key encoded value change time 876 | open_cursor(C, Tab) -> 877 | CursorName = "cursor_" ++ Tab ++ integer_to_list(mnesia_pg_conns:ref()), 878 | {ok, [], []} = pgsql:squery(C, "begin"), 879 | {ok, [], []} = pgsql:squery(C, ["declare ", CursorName, " cursor for select erlkey,erlval from ", Tab, " order by erlkey asc"]), 880 | {C, CursorName}. 881 | 882 | open_cursor(C, Tab, Pat) -> 883 | CursorName = "cursor_" ++ Tab ++ integer_to_list(mnesia_pg_conns:ref()), 884 | {ok, [], []} = pgsql:squery(C, "begin"), 885 | {ok, [], []} = pgsql:squery(C, ["declare ", CursorName, " cursor for select erlkey,erlval from ", Tab, " where erlkey >= '", Pat, "' order by erlkey asc"]), 886 | {C, CursorName}. 887 | 888 | commit_cursor({C, CursorName}) -> 889 | pgsql:squery(C, ["close ", CursorName]), 890 | pgsql:squery(C, "commit"). 891 | 892 | rollback_cursor({C, CursorName}) -> 893 | pgsql:squery(C, ["close ", CursorName]), 894 | pgsql:squery(C, "rollback"). 895 | 896 | fetch_next({C, CursorName}) -> 897 | Res = pgsql:equery(C, ["fetch next from ", CursorName], []), 898 | case (Res) of 899 | {ok, 0} -> 900 | void; 901 | {ok, 1, _, [R]} -> 902 | {ok, element(1, R), element(2, R)} 903 | end. 904 | 905 | fetch_next_n(C) -> 906 | fetch_next_n(1, C). 907 | 908 | fetch_next_n(N, {C, CursorName}) -> 909 | Res = pgsql:equery(C, ["fetch ", integer_to_list(N), 910 | " from ", CursorName], []), 911 | case Res of 912 | {ok, 0} -> 913 | []; 914 | {ok, _, _, Rows} -> 915 | [{element(1, R), element(2, R)} || R <- Rows] 916 | end. 917 | 918 | %pgsql:equery(C, "insert into " ++ Tab ++ " (erlkey,erlval,change_time) values ($1,$2,CURRENT_TIMESTAMP)", [PKey,Val]), 919 | upsert_into(Alias, Tab0, PKey, SKey, Val) -> 920 | {C, Tab} = get_ref(Alias, Tab0), 921 | try 922 | {ok, _, _} = pgsql:equery(C, ["select upsert_", Tab, "($1,$2,$3)"], [PKey,SKey,Val]), 923 | ok 924 | after 925 | mnesia_pg_conns:free(C) 926 | end. 927 | 928 | delete_from(Alias, Tab0, PKey) -> 929 | {C, Tab} = get_ref(Alias, Tab0), 930 | try 931 | {ok, N} = pgsql:equery(C, ["delete from ", Tab, " where erlkey=$1"], [PKey]), 932 | N 933 | after 934 | mnesia_pg_conns:free(C) 935 | end. 936 | 937 | delete_list(Alias, Tab0, PKeyList) -> 938 | {C, Tab} = get_ref(Alias, Tab0), 939 | try 940 | N = lists:foldl(fun(PKey, Sum) -> 941 | {ok, N} = pgsql:equery(C, ["delete from ", Tab, " where erlkey=$1"], [PKey]), 942 | Sum + N 943 | end, 944 | 0, 945 | PKeyList), 946 | N 947 | after 948 | mnesia_pg_conns:free(C) 949 | end. 950 | 951 | delete_all(Alias, Tab0) -> 952 | {C, Tab} = get_ref(Alias, Tab0), 953 | try 954 | pgsql:squery(C, ["delete from ", Tab]) 955 | after 956 | mnesia_pg_conns:free(C) 957 | end. 958 | 959 | delete_from_pattern(Alias, Tab0, Pat) -> 960 | {C, _} = Ref = get_ref(Alias, Tab0), 961 | Fun = fun(_, Key, Acc) -> [Key|Acc] end, 962 | try 963 | Keys = do_fold(Ref, Tab0, Fun, [], [{Pat,[],['$_']}], 30), 964 | if Keys == [] -> 965 | ok; 966 | true -> 967 | delete_list(Alias, Ref, Keys), 968 | ok 969 | end 970 | after 971 | mnesia_pg_conns:free(C) 972 | end. 973 | 974 | %% TODO: check that Tab syntax is compatible with postgresql 975 | create_table(Alias, Tab0, _Props) -> 976 | {C, Tab} = get_ref(Alias, Tab0), 977 | try 978 | pgsql:squery(C, ["drop function if exists upsert_", Tab, "(ekey character varying, ehash bigint, eval bytea)"]), 979 | pgsql:squery(C, ["drop index if exists ", Tab, "_term_idx"]), 980 | pgsql:squery(C, ["drop table if exists ", Tab]), 981 | {ok, [], []} = pgsql:squery(C, ["create table ", Tab, "( erlkey character varying(2048) primary key, erlhash bigint not null, erlval bytea not null, change_time timestamp not null )"]), 982 | {ok, [], []} = pgsql:squery(C, ["create function upsert_", Tab, "(ekey character varying, ehash bigint, eval bytea) RETURNS void AS\n", 983 | "$$\n", "BEGIN\n", " INSERT INTO ", Tab, " VALUES (ekey, ehash, eval, current_timestamp);\n", 984 | "EXCEPTION WHEN unique_violation THEN\n", 985 | " UPDATE ", Tab, " SET erlval=eval, erlhash=ehash, change_time=current_timestamp WHERE erlkey = ekey;\n", 986 | "END;\n", "$$\n", "LANGUAGE plpgsql;"]), 987 | {ok, [], []} = pgsql:squery(C, ["create index ", Tab, "_term_idx on " ++ Tab ++ " using hash (erlhash)"]), 988 | ok 989 | after 990 | mnesia_pg_conns:free(C) 991 | end. 992 | 993 | delete_table(Alias, Tab0) -> 994 | {C, Tab} = get_ref(Alias, Tab0), 995 | try 996 | pgsql:squery(C, ["drop function if exists upsert_", Tab, "(ekey character varying, ehash bigint, eval bytea)"]), 997 | pgsql:squery(C, ["drop table if exists ", Tab]) 998 | after 999 | mnesia_pg_conns:free(C) 1000 | end. 1001 | 1002 | load_table(_Alias, _Tab, _LoadReason, _Opts) -> 1003 | ok. 1004 | 1005 | close_table(_Alias, _Tab) -> 1006 | ok. 1007 | 1008 | table_count(Alias, Tab0) -> 1009 | {C, Tab} = get_ref(Alias, Tab0), 1010 | try 1011 | {ok, _, [{X}]} = pgsql:equery(C, ["select count(*) from ", Tab], []), 1012 | X 1013 | catch 1014 | error:_ -> 1015 | 0 1016 | after 1017 | mnesia_pg_conns:free(C) 1018 | end. 1019 | 1020 | table_size(Alias, Tab0) -> 1021 | {C, Tab} = get_ref(Alias, Tab0), 1022 | try 1023 | {ok, _, [{X}]} = pgsql:equery(C, ["select pg_total_relation_size('", Tab, "')"], []), 1024 | X div 4 % word definition of mnesia? 1025 | catch 1026 | error:_ -> 1027 | 1 1028 | after 1029 | mnesia_pg_conns:free(C) 1030 | end. 1031 | 1032 | get_ref(_Alias, Tab) -> 1033 | C = mnesia_pg_conns:alloc(Tab), 1034 | {C, tabname(Tab)}. 1035 | 1036 | tabname({Tab, index, {{Pos},_}}) -> 1037 | "ix2_" ++ atom_to_list(Tab) ++ "_" ++ atom_to_list(Pos); 1038 | tabname({Tab, index, {Pos,_}}) -> 1039 | "ix_" ++ atom_to_list(Tab) ++ "_" ++ integer_to_list(Pos); 1040 | tabname({Tab, retainer, Name}) -> 1041 | "ret_" ++ atom_to_list(Tab) ++ "_" ++ retainername(Name); 1042 | tabname(Tab) when is_atom(Tab) -> 1043 | "tab_" ++ atom_to_list(Tab). 1044 | 1045 | retainername(Name) when is_atom(Name) -> 1046 | atom_to_list(Name); 1047 | retainername(Name) when is_list(Name) -> 1048 | try binary_to_list(list_to_binary(Name)) 1049 | catch 1050 | error:_ -> 1051 | lists:flatten(io_lib:write(Name)) 1052 | end; 1053 | retainername(Name) -> 1054 | lists:flatten(io_lib:write(Name)). 1055 | 1056 | dbg(on) -> 1057 | dbg:start(), 1058 | dbg:tracer(), 1059 | dbg:tpl(mnesia, '_', []), 1060 | dbg:tpl(mnesia_pg, '_', []), 1061 | dbg:tpl(mnesia_pg_conns, '_', []), 1062 | dbg:tpl(mnesia_pg_sup, '_', []), 1063 | dbg:tpl(mnesia_pg_app, '_', []), 1064 | dbg:tpl(pgsql, '_', []), 1065 | dbg:p(all,c); 1066 | dbg(off) -> 1067 | dbg:stop_clear(). 1068 | 1069 | %%%impl_doc_include 1070 | -------------------------------------------------------------------------------- /src/mnesia_pg_app.erl: -------------------------------------------------------------------------------- 1 | %%% 2 | %%% Copyright (c) 2014-2015 Klarna AB 3 | %%% 4 | %%% This file is provided to you under the Apache License, 5 | %%% Version 2.0 (the "License"); you may not use this file 6 | %%% except in compliance with the License. You may obtain 7 | %%% a copy of the License at 8 | %%% 9 | %%% http://www.apache.org/licenses/LICENSE-2.0 10 | %%% 11 | %%% Unless required by applicable law or agreed to in writing, 12 | %%% software distributed under the License is distributed on an 13 | %%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | %%% KIND, either express or implied. See the License for the 15 | %%% specific language governing permissions and limitations 16 | %%% under the License. 17 | 18 | -module(mnesia_pg_app). 19 | 20 | -behaviour(application). 21 | 22 | %% Application callbacks 23 | -export([start/2, 24 | stop/1, 25 | start_phase/3]). 26 | 27 | %% =================================================================== 28 | %% Application callbacks 29 | %% =================================================================== 30 | 31 | start(_StartType, _StartArgs) -> 32 | mnesia_pg_sup:start_link(). 33 | 34 | stop(_State) -> 35 | ok. 36 | 37 | start_phase(check_schema_cookie, _, _) -> 38 | case mnesia_pg_conns:check_schema_cookie() of 39 | ok -> 40 | ok; 41 | Error -> 42 | Error 43 | end. 44 | -------------------------------------------------------------------------------- /src/mnesia_pg_conns.erl: -------------------------------------------------------------------------------- 1 | %%% 2 | %%% Copyright (c) 2014-2015 Klarna AB 3 | %%% 4 | %%% This file is provided to you under the Apache License, 5 | %%% Version 2.0 (the "License"); you may not use this file 6 | %%% except in compliance with the License. You may obtain 7 | %%% a copy of the License at 8 | %%% 9 | %%% http://www.apache.org/licenses/LICENSE-2.0 10 | %%% 11 | %%% Unless required by applicable law or agreed to in writing, 12 | %%% software distributed under the License is distributed on an 13 | %%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | %%% KIND, either express or implied. See the License for the 15 | %%% specific language governing permissions and limitations 16 | %%% under the License. 17 | 18 | -module(mnesia_pg_conns). 19 | 20 | -behaviour(gen_server). 21 | -export([start_link/0]). 22 | -export([alloc/1, free/1, ref/0, state/0]). 23 | -export([check_schema_cookie/0]). 24 | -export([init/1, handle_call/3, handle_cast/2, 25 | handle_info/2, code_change/3, terminate/2]). 26 | 27 | -include("mnesia_pg_int.hrl"). 28 | 29 | start_link() -> 30 | Conf = mnesia_pgsql_mon:get_conf(), 31 | gen_server:start_link({local, conn_pool}, ?MODULE, Conf, []). 32 | 33 | open_connections(#conf{pool_size = N, 34 | host = Host, 35 | port = Port, 36 | user = User, 37 | password = Pwd, 38 | db = Db}) -> 39 | L = lists:seq(1, N), % N connections 40 | ConnL = lists:map( 41 | fun(_) -> 42 | {ok, C} = pgsql:connect(Host, User, Pwd, 43 | [{database, Db}, 44 | {port, Port}]), 45 | C 46 | end, L), 47 | ConnL. 48 | 49 | alloc(Tab) -> 50 | C = gen_server:call(conn_pool, {alloc, self()}), 51 | case (C) of 52 | wait -> 53 | receive 54 | {new_conn, C2} -> 55 | C2 56 | after 1000 -> 57 | io:fwrite("ERR: failed to allocate PG connection for table ~p~n", [Tab]), 58 | void % mnesia abort? 59 | end; 60 | _ -> 61 | C 62 | end. 63 | 64 | free(Conn) -> 65 | gen_server:cast(conn_pool, {free, Conn}). 66 | 67 | ref() -> 68 | gen_server:call(conn_pool, get_ref). 69 | 70 | state() -> 71 | gen_server:call(conn_pool, get_state). 72 | 73 | check_schema_cookie() -> 74 | gen_server:call(conn_pool, check_schema_cookie). 75 | 76 | init(Conf) -> 77 | ConnL = open_connections(Conf), 78 | io:fwrite("Connection opened~n", []), 79 | %% check_schema_cookie(ConnL), 80 | process_flag(trap_exit, true), 81 | {A1,A2,A3} = now(), 82 | random:seed(A1, A2, A3), 83 | {ok, {ConnL, []}}. 84 | 85 | terminate(_, {ConnL, _}) -> 86 | lists:foreach(fun(C) -> 87 | try pgsql:close(C) 88 | catch 89 | _:_ -> ok 90 | end 91 | end, ConnL). 92 | 93 | handle_call(get_ref, _From, State) -> 94 | {reply, random:uniform(100000000), State}; 95 | handle_call(get_state, _From, State) -> 96 | {reply, State, State}; 97 | handle_call({alloc, _Pid}, _From, {[Conn|ConnL], WaitL}) -> 98 | {reply, Conn, {ConnL, WaitL}}; 99 | handle_call({alloc, Pid}, _From, {[], WaitL}) -> 100 | {reply, wait, {[], lists:append(WaitL,[Pid])}}; 101 | handle_call(check_schema_cookie, _From, {ConnL, _} = State) -> 102 | Result = check_schema_cookie(ConnL), 103 | {reply, Result, State}. 104 | 105 | handle_cast({free, Conn}, {ConnL, []}) -> 106 | {noreply, {[Conn|ConnL], []}}; 107 | handle_cast({free, Conn}, {ConnL, [Pid|WaitL]}) -> 108 | Pid ! {new_conn, Conn}, 109 | {noreply, {ConnL, WaitL}}. 110 | 111 | handle_info(_, State) -> 112 | {noreply, State}. 113 | 114 | code_change(_OldVsn, State, _Extra) -> 115 | {ok, State}. 116 | 117 | %% Internal 118 | 119 | check_schema_cookie([H|_]) -> 120 | sql_transaction(H, fun() -> do_check_schema_cookie(H) end). 121 | 122 | do_check_schema_cookie(H) -> 123 | MyCookie = mnesia_lib:val({schema, cookie}), 124 | SQL = "select erlval from schema where erlkey='cookie'", 125 | Res = pgsql:equery(H, SQL, []), 126 | io:fwrite("schema query: ~p~n", [Res]), 127 | case Res of 128 | {error,{error,error,<<"42P01">>,_,_}} -> 129 | %% schema doesn't exist 130 | io:fwrite("schema doesn't exist~n", []), 131 | CreateRes = sql_create_schema(H), 132 | io:fwrite("CreateRes = ~p~n", [CreateRes]), 133 | insert_cookie(H, MyCookie); 134 | {ok, _, []} -> 135 | %% No cookie 136 | io:fwrite("No cookie~n", []), 137 | insert_cookie(H, MyCookie), 138 | ok; 139 | {ok, [{column,<<"erlval">>,bytea,_,_,_}],[{Bin}]} = Res -> 140 | io:fwrite("Res = ~p~n", [Res]), 141 | try binary_to_term(Bin) of 142 | MyCookie -> 143 | io:fwrite("Cookies match!~n", []), 144 | ok; 145 | WrongCookie -> 146 | {error, 147 | {schema_cookie_mismatch,{WrongCookie,MyCookie}}} 148 | catch 149 | error:_ -> 150 | error(cannot_decode_cookie) 151 | end 152 | end. 153 | 154 | insert_cookie(C, Cookie) -> 155 | Bin = term_to_binary(Cookie), 156 | InsertRes = 157 | pgsql:equery(C, ("insert into schema (erlkey, erlval) values" 158 | " ('cookie', $1)"), [Bin]), 159 | io:fwrite("InsertRes = ~p~n", [InsertRes]), 160 | File = filename:join(mnesia_monitor:get_env(dir), "cookie.pg"), 161 | file:write_file(File, Bin), 162 | ok. 163 | 164 | 165 | 166 | sql_transaction(_C, F) -> 167 | %% pgsql:squery(C, "begin"), 168 | try F() 169 | %% pgsql:squery(C, "commit") 170 | catch 171 | error:E -> 172 | %% pgsql:squery(C, "rollback"), 173 | error(E); 174 | exit:R -> 175 | %% pgsql:squery(C, "rollback"), 176 | exit(R); 177 | throw:T -> 178 | %% pgsql:squery(C, "rollback"), 179 | throw(T) 180 | end. 181 | 182 | 183 | 184 | sql_create_schema(H) -> 185 | pgsql:squery(H, ("create table schema" 186 | " (erlkey character varying(64), erlval bytea)")). 187 | -------------------------------------------------------------------------------- /src/mnesia_pg_int.hrl: -------------------------------------------------------------------------------- 1 | %%% -*- erlang-indent-level: 4; indent-tabs-mode: nil -*- 2 | %%% 3 | %%% Copyright (c) 2014-2015 Klarna AB 4 | %%% 5 | %%% This file is provided to you under the Apache License, 6 | %%% Version 2.0 (the "License"); you may not use this file 7 | %%% except in compliance with the License. You may obtain 8 | %%% a copy of the License at 9 | %%% 10 | %%% http://www.apache.org/licenses/LICENSE-2.0 11 | %%% 12 | %%% Unless required by applicable law or agreed to in writing, 13 | %%% software distributed under the License is distributed on an 14 | %%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %%% KIND, either express or implied. See the License for the 16 | %%% specific language governing permissions and limitations 17 | %%% under the License. 18 | 19 | -record(conf, {status, 20 | was_running = false, 21 | bin, 22 | dir, 23 | host = "localhost", 24 | port = 5432, 25 | user = "mnesia", 26 | password = "", 27 | pool_size = 10, 28 | db = "mnesia", 29 | tablespace}). 30 | -------------------------------------------------------------------------------- /src/mnesia_pg_sup.erl: -------------------------------------------------------------------------------- 1 | %%% 2 | %%% Copyright (c) 2014-2015 Klarna AB 3 | %%% 4 | %%% This file is provided to you under the Apache License, 5 | %%% Version 2.0 (the "License"); you may not use this file 6 | %%% except in compliance with the License. You may obtain 7 | %%% a copy of the License at 8 | %%% 9 | %%% http://www.apache.org/licenses/LICENSE-2.0 10 | %%% 11 | %%% Unless required by applicable law or agreed to in writing, 12 | %%% software distributed under the License is distributed on an 13 | %%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | %%% KIND, either express or implied. See the License for the 15 | %%% specific language governing permissions and limitations 16 | %%% under the License. 17 | 18 | -module(mnesia_pg_sup). 19 | 20 | -behaviour(supervisor). 21 | 22 | %% API 23 | -export([start_link/0]). 24 | 25 | %% Supervisor callbacks 26 | -export([init/1]). 27 | 28 | %% Helper macro for declaring children of supervisor 29 | -define(CHILD(I, Type), {I, {I, start_link, []}, permanent, 5000, Type, [I]}). 30 | 31 | %% =================================================================== 32 | %% API functions 33 | %% =================================================================== 34 | 35 | start_link() -> 36 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 37 | 38 | %% =================================================================== 39 | %% Supervisor callbacks 40 | %% =================================================================== 41 | 42 | init([]) -> 43 | {ok, { {one_for_one, 5, 10}, [?CHILD(mnesia_pgsql_mon, worker), 44 | ?CHILD(mnesia_pg_conns, worker)]} }. 45 | -------------------------------------------------------------------------------- /src/mnesia_pg_util.erl: -------------------------------------------------------------------------------- 1 | %%% 2 | %%% Copyright (c) 2014-2015 Klarna AB 3 | %%% 4 | %%% This file is provided to you under the Apache License, 5 | %%% Version 2.0 (the "License"); you may not use this file 6 | %%% except in compliance with the License. You may obtain 7 | %%% a copy of the License at 8 | %%% 9 | %%% http://www.apache.org/licenses/LICENSE-2.0 10 | %%% 11 | %%% Unless required by applicable law or agreed to in writing, 12 | %%% software distributed under the License is distributed on an 13 | %%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | %%% KIND, either express or implied. See the License for the 15 | %%% specific language governing permissions and limitations 16 | %%% under the License. 17 | 18 | -module(mnesia_pg_util). 19 | 20 | -export([init_env/0, 21 | create/0, 22 | pg_dir/1, 23 | pg_bin/1, 24 | get_env/2, 25 | stop/0, stop/2, 26 | get_saved_port/0]). 27 | -export([clear_psql_db/0]). 28 | -export([test/0]). 29 | -export([ttest/0]). 30 | 31 | -include("mnesia_pg_int.hrl"). 32 | 33 | init_env() -> 34 | C0 = #conf{}, 35 | application:load(mnesia_pg), 36 | User = get_env(pg_user, C0#conf.user), 37 | Pwd = get_env(pg_pwd, C0#conf.password), 38 | Db = get_env(pg_db, C0#conf.db), 39 | Pool = get_env(pool_size, C0#conf.pool_size), 40 | PgBin = get_env(pg_bin, fun get_bin/0), 41 | Dir = mnesia_monitor:get_env(dir), 42 | PgDir = get_env(pg_dir, filename:join(Dir, "pgdata")), 43 | C = #conf{bin = PgBin, 44 | dir = PgDir, 45 | db = Db, 46 | user = User, 47 | password = Pwd, 48 | pool_size = Pool}, 49 | case filelib:is_dir(PgDir) of 50 | false -> C#conf{status = not_installed, 51 | port = get_port()}; 52 | true -> pg_status(C) 53 | end. 54 | 55 | clear_psql_db() -> 56 | case mnesia_lib:is_running() of 57 | no -> 58 | application:load(mnesia), 59 | case init_env() of 60 | #conf{status = running, was_running = true} = C -> 61 | psql_drop_all_tables(C), 62 | ok; 63 | _ -> 64 | {error, postgres_not_running} 65 | end; 66 | _ -> 67 | {error, mnesia_is_running} 68 | end. 69 | 70 | pg_dir(#conf{dir = Dir}) -> 71 | Dir. 72 | 73 | pg_bin(#conf{bin = Bin}) -> 74 | Bin. 75 | 76 | 77 | ttest() -> 78 | dbg:tracer(), 79 | dbg:tpl(?MODULE, x), 80 | dbg:p(all,[c]), 81 | test(). 82 | 83 | test() -> 84 | mnesia:stop(), 85 | %% stop(), 86 | mnesia:delete_schema([node()]), 87 | mnesia:create_schema([node()], [{backend_types, [{pg, mnesia_pg}]}]), 88 | create(). 89 | 90 | create() -> 91 | C = init_env(), 92 | io:fwrite("init_env() -> ~p~n", [C]), 93 | ensure_running(C). 94 | 95 | ensure_running(#conf{status = not_installed} = C) -> 96 | start_db(init_db(C)); 97 | ensure_running(#conf{status = not_running} = C) -> 98 | start_db(C); 99 | ensure_running(#conf{status = running} = C) -> 100 | C. 101 | 102 | init_db(#conf{status = not_installed, 103 | bin = PgBin, dir = PgDir} = C) -> 104 | cmd([filename:join(PgBin,"initdb"), " -D ", PgDir]), 105 | C#conf{status = not_running}. 106 | 107 | start_db(#conf{status = not_running, dir = PgDir, 108 | port = Port, user = User, db = Db} = C) -> 109 | PortStr = integer_to_list(Port), 110 | PgLog = filename:join(mnesia_monitor:get_env(dir), "pglog"), 111 | Opts = "-i -h localhost -p " ++ PortStr, 112 | cmd([c("pg_ctl", C), " start -D ", PgDir, " -l ", PgLog, 113 | " -w -t 10 -o \"", Opts, "\""]), 114 | CreateUser = 115 | cmd([c("createuser",C), " -h localhost -p ", PortStr, 116 | " ", User]), 117 | cmd([c("createdb",C), " -h localhost -p ", PortStr, 118 | " --owner=", User, " ", Db]), 119 | modify_user(CreateUser, C), 120 | save_port(Port), 121 | C#conf{status = running, was_running = false}. 122 | 123 | modify_user(_, #conf{password = ""}) -> 124 | ok; 125 | modify_user(Res, #conf{user = User, password = Pwd} = C) -> 126 | case Res of 127 | "createuser: " ++ _ = Err -> 128 | case re:run(Err, "already exists", []) of 129 | {match, _} -> ok; 130 | _ -> error({unknown_error, Err}) 131 | end; 132 | _ -> ok 133 | end, 134 | psql(["alter role ", User, " with password '", Pwd, "';"], C). 135 | 136 | 137 | %% psql(SQL) -> 138 | %% Conf = mnesia_pgsql_mon:get_conf(), 139 | %% psql(SQL, Conf). 140 | 141 | psql(SQL, #conf{bin = PgBin, db = Db, host = Host, port = Port}) -> 142 | PortStr = integer_to_list(Port), 143 | cmd([filename:join(PgBin, "psql"), " -h ", Host, " -p ", PortStr, 144 | " -d ", Db, " -c \"", SQL, "\""]). 145 | 146 | psql_drop_all_tables(C) -> 147 | Tabs = psql_list_tables(C), 148 | io:fwrite("list_tables -> ~p~n", [Tabs]), 149 | [psql(["drop table if exists \"", T, "\" cascade"], C) 150 | || T <- Tabs]. 151 | 152 | psql_list_tables(#conf{bin = PgBin, db = Db, host = Host, port = Port}) -> 153 | PortStr = integer_to_list(Port), 154 | SQL = ("select table_name from information_schema.tables" 155 | " where table_schema='public'"), 156 | R = cmd([filename:join(PgBin, "psql"), " -h ", Host, " -p ", PortStr, 157 | " -d ", Db, " -t -c \"", SQL, "\""]), 158 | string:tokens(R, "\n\r\t\s"). 159 | 160 | 161 | c(Cmd, #conf{bin = Bin}) -> 162 | filename:join(Bin, Cmd). 163 | 164 | stop() -> 165 | Dir = mnesia_monitor:get_env(dir), 166 | PgDir = filename:join(Dir, "pgdata"), 167 | PgBin = get_env(pg_bin, "/usr/local/pgsql/bin"), 168 | stop(PgBin, PgDir). 169 | 170 | stop(PgBin, PgDir) -> 171 | cmd([filename:join(PgBin,"pg_ctl"), "stop -D ", PgDir]). 172 | 173 | cmd(Cmd0) -> 174 | Cmd = lists:flatten(Cmd0), 175 | L = erlang:min(length(Cmd), 50), 176 | Res = os:cmd(Cmd), 177 | io:fwrite("~s~n" ++ lists:duplicate(L, $-) 178 | ++ "~n~s~n", [Cmd, Res]), 179 | case Res of 180 | "/bin/sh:" ++ _ -> 181 | error(script_error); 182 | _ -> 183 | Res 184 | end. 185 | 186 | pg_status(#conf{bin = Bin, dir = Dir} = C) -> 187 | Ctl = filename:join(Bin, "pg_ctl status"), 188 | Cmd = [Ctl, " -D ", Dir], 189 | if_not_running(parse_status(cmd(Cmd), C)). 190 | 191 | parse_status("pg_ctl: no server" ++ _, C) -> 192 | C#conf{status = not_running}; 193 | parse_status("pg_ctl: server is running" ++ Rest, C) -> 194 | Pid = re:run(Rest, "\\(PID: ([0-9]+)\\)", [{capture,[1], list}]), 195 | io:fwrite("postgres running at (PID: ~p)~n", [Pid]), 196 | [_,Cmd|_] = re:split(Rest, "\\n", [{return,list}]), 197 | CmdS = re:replace(Cmd, "\\\"", "", [global]), 198 | case re:run(CmdS, "-i", []) of 199 | {match,_} -> 200 | %% TCP enabled 201 | H = match1(re:run(CmdS, "-h[\\h]+([^\\h]+)", [{capture,[1],list}])), 202 | P = match1(re:run(CmdS, "-p[\\h]+([^\\h]+)", [{capture,[1],list}])), 203 | C#conf{status = running, 204 | host = H, 205 | port = list_to_integer(P), 206 | was_running = true}; 207 | nomatch -> 208 | error(inet_not_enabled) 209 | end. 210 | 211 | if_not_running(#conf{status = not_running} = C) -> 212 | P = get_port(), 213 | H = get_env(pg_host, "localhost"), 214 | C#conf{host = H, 215 | port = P}; 216 | if_not_running(C) -> 217 | C. 218 | 219 | match1({match, [Res]}) -> Res; 220 | match1(nomatch) -> undefined. 221 | 222 | get_env(K, Default) -> 223 | case application:get_env(mnesia_pg, K) of 224 | {ok, Val} -> 225 | Val; 226 | _ -> 227 | if is_function(Default, 0) -> Default(); 228 | true -> Default 229 | end 230 | end. 231 | 232 | get_bin() -> 233 | RelPath = "pgsql/bin", 234 | case code:lib_dir(mnesia_pg) of 235 | {error, bad_name} -> 236 | Dir = filename:join( 237 | filename:dirname( 238 | filename:dirname( 239 | filename:absname(code:which(?MODULE)))), RelPath), 240 | case filelib:is_regular( 241 | filename:join(Dir, "psql")) of 242 | false -> 243 | error(cannot_determine_bin_dir); 244 | true -> 245 | Dir 246 | end; 247 | Lib -> 248 | filename:join(Lib, RelPath) 249 | end. 250 | 251 | get_port() -> 252 | get_env(pg_port, fun any_port/0). 253 | 254 | %% This doesn't guarantee that the port will be available, but the OS seems 255 | %% reluctant to reuse port numbers right away (for good reason). 256 | any_port() -> 257 | {ok, S} = gen_tcp:listen(0, []), 258 | {ok, P} = inet:port(S), 259 | gen_tcp:close(S), 260 | application:set_env(mnesia_pg, pg_port, P), 261 | P. 262 | 263 | save_port(Port) -> 264 | Dir = mnesia_monitor:get_env(dir), 265 | Bin = term_to_binary([{port, Port}]), 266 | file:write_file(filename:join(Dir, "port_info.pg"), Bin). 267 | 268 | get_saved_port() -> 269 | Dir = mnesia_monitor:get_env(dir), 270 | case file:read_file(filename:join(Dir, "port_info.pg")) of 271 | {ok, Bin} -> 272 | proplists:get_value(port, binary_to_term(Bin)); 273 | _ -> 274 | undefined 275 | end. 276 | -------------------------------------------------------------------------------- /src/mnesia_pgsql_mon.erl: -------------------------------------------------------------------------------- 1 | %%% -*- erlang-indent-level: 4; indent-tabs-mode: nil -*- 2 | %%% 3 | %%% Copyright (c) 2014-2015 Klarna AB 4 | %%% 5 | %%% This file is provided to you under the Apache License, 6 | %%% Version 2.0 (the "License"); you may not use this file 7 | %%% except in compliance with the License. You may obtain 8 | %%% a copy of the License at 9 | %%% 10 | %%% http://www.apache.org/licenses/LICENSE-2.0 11 | %%% 12 | %%% Unless required by applicable law or agreed to in writing, 13 | %%% software distributed under the License is distributed on an 14 | %%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %%% KIND, either express or implied. See the License for the 16 | %%% specific language governing permissions and limitations 17 | %%% under the License. 18 | 19 | -module(mnesia_pgsql_mon). 20 | 21 | -behaviour(gen_server). 22 | 23 | -export([get_conf/0]). 24 | 25 | -export([connect/1]). 26 | 27 | -record(st, {master, 28 | mref, 29 | pg_bin, 30 | pg_dir, 31 | port, 32 | conf, 33 | sock}). 34 | 35 | -include("mnesia_pg_int.hrl"). 36 | 37 | -record(srv, {lsock, 38 | sock, 39 | port, 40 | conf}). % server state 41 | 42 | %% gen_server API 43 | -export([start_link/0, 44 | init/1, 45 | handle_call/3, 46 | handle_cast/2, 47 | handle_info/2, 48 | terminate/2, 49 | code_change/3]). 50 | 51 | get_conf() -> 52 | gen_server:call(?MODULE, get_conf). 53 | 54 | start_link() -> 55 | gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). 56 | 57 | init([]) -> 58 | {ok, LSock} = gen_tcp:listen(0, mon_port_opts()), 59 | {ok, MyPort} = inet:port(LSock), 60 | Me = self(), 61 | Conf = mnesia_pg_util:create(), 62 | io:fwrite("init: Conf = ~p (was_running=~p)~n", [Conf,Conf#conf.was_running]), 63 | case Conf#conf.was_running of 64 | true -> 65 | ok; 66 | false -> 67 | %% We started a postgres instance; monitor it! 68 | spawn_link(fun() -> do_accept(LSock, Me) end), 69 | spawn_link(fun() -> start(MyPort) end) 70 | end, 71 | spawn_link(fun monitor_mnesia/0), 72 | {ok, #srv{lsock = LSock, 73 | port = MyPort, 74 | conf = Conf}}. 75 | 76 | 77 | handle_cast(_, S) -> 78 | {noreply, S}. 79 | 80 | handle_info({accepted, Sock}, #srv{conf = Conf} = S) -> 81 | io:fwrite("got {accepted, ~p}~n", [Sock]), 82 | SendRes = gen_tcp:send(Sock, term_to_binary(Conf)), 83 | io:fwrite("SendRes = ~p~n", [SendRes]), 84 | {noreply, S#srv{sock = Sock}}; 85 | handle_info({tcp_closed, Sock}, #srv{sock = Sock} = S) -> 86 | io:fwrite("Got tcp_closed (~p)~n", [Sock]), 87 | {stop, remote_closed, S}; 88 | handle_info(Msg, S) -> 89 | io:fwrite("Got unknown Msg = ~p~n", [Msg]), 90 | {noreply, S}. 91 | 92 | handle_call(get_conf, _, #srv{conf = Conf} = S) -> 93 | {reply, Conf, S}; 94 | handle_call(_, _, S) -> {reply, error, S}. 95 | terminate(_, _) -> ok. 96 | code_change(_, S, _) -> {ok, S}. 97 | 98 | do_accept(LSock, Parent) -> 99 | io:fwrite("do_accept(~p - ~p)~n", [LSock, inet:port(LSock)]), 100 | case gen_tcp:accept(LSock, 30000) of 101 | {ok, Sock} -> 102 | io:fwrite("pgsql monitor connected!~n", []), 103 | ok = gen_tcp:controlling_process(Sock, Parent), 104 | io:fwrite("Parent (~p) now controlling~n", [Parent]), 105 | Parent ! {accepted, Sock}; 106 | Error -> 107 | exit({accept_error, Error}) 108 | end. 109 | 110 | monitor_mnesia() -> 111 | MRef = monitor(process, mnesia_sup), 112 | receive 113 | {'DOWN', MRef, _, _, _} -> 114 | application:stop(mnesia_pg) 115 | end. 116 | 117 | start(MyPort) -> 118 | Ebin = filename:dirname(code:which(mnesia_pg)), 119 | io:fwrite("Ebin = ~p~n", [Ebin]), 120 | Log = filename:join(mnesia_monitor:get_env(dir), "mon_log.pg"), 121 | EnsureRes = filelib:ensure_dir(filename:join(Log, "dummy")), 122 | io:fwrite("Ensure = ~p~n", [EnsureRes]), 123 | Cmd = ["run_erl /tmp/ulf/ ", Log, " \"erl -pa ", Ebin, 124 | " -run mnesia_pgsql_mon connect ", integer_to_list(MyPort), 125 | "\""], 126 | io:fwrite("Cmd = ~s~n", [Cmd]), 127 | CmdRes = os:cmd(Cmd), 128 | io:fwrite("CmdRes = ~p~n", [CmdRes]), 129 | ok. 130 | 131 | connect([PortS]) -> 132 | spawn(fun() -> do_connect(list_to_integer(PortS)) end). 133 | 134 | do_connect(Port) -> 135 | case gen_tcp:connect("localhost", Port, mon_port_opts()) of 136 | {ok, Sock} -> 137 | io:fwrite("Connected to master~n", []), 138 | receive 139 | {tcp, Sock, Cmd} -> 140 | run(binary_to_term(Cmd), Sock); 141 | {tcp_closed, Sock} -> 142 | error(tcp_closed); 143 | Other -> 144 | io:fwrite("Received Other: ~p~n", [Other]), 145 | error(protocol_error) 146 | after 10000 -> 147 | error(timeout) 148 | end; 149 | Other -> 150 | error({connect_error, Other}) 151 | end. 152 | 153 | 154 | run(Conf, Sock) -> 155 | io:fwrite("run(~p)~n", [Conf]), 156 | loop(#st{conf = Conf, sock = Sock}). 157 | 158 | loop(#st{sock = Sock, conf = Conf} = S) -> 159 | receive 160 | {tcp_closed, Sock} -> 161 | io:fwrite("sock (~p) closed~n", [Sock]), 162 | stop_pgsql(Conf), 163 | init:stop(); 164 | Other -> 165 | io:fwrite("Received ~p - ignoring~n", [Other]), 166 | loop(S) 167 | end. 168 | 169 | stop_pgsql(#conf{bin = PgBin, dir = PgDir}) -> 170 | mnesia_pg_util:stop(PgBin, PgDir). 171 | 172 | mon_port_opts() -> 173 | [binary, 174 | {active, true}, 175 | {delay_send, false}, 176 | {packet, 4}]. 177 | -------------------------------------------------------------------------------- /stdapp.mk: -------------------------------------------------------------------------------- 1 | # Makefile for building an Erlang application 2 | # Usage: make -C -f stdapp.mk [target] 3 | # 4 | # Targets: 5 | # build 6 | # tests 7 | # docs 8 | # clean 9 | # distclean 10 | # realclean 11 | # clean-tests 12 | # clean-docs 13 | # 14 | # Running 'make -f stdapp.mk' in an empty directory will create the file 15 | # src/APPLICATION.app.src, taking the application name from the directory 16 | # name. You can override this using 'make APPLICATION=... -f stdapp.mk'. 17 | # Once the src/*.app.src (or ebin/*.app) file exists, the name of that file 18 | # will be used for the application name. When compiling an Erlang program 19 | # using stdapp.mk, the macro ?APPLICATION will be automatically defined. You 20 | # can override the name of this macro to avoid collisions by setting the 21 | # make variable APPLICATION_NAME_MACRO. 22 | # 23 | # The following is an example of a minimal top level Makefile for building 24 | # all applications in the lib/ subdirectory: 25 | # 26 | # TOP_DIR = $(CURDIR) 27 | # APPS = $(wildcard lib/*) 28 | # BUILD_TARGETS = $(APPS:lib/%=build-%) 29 | # .PHONY: all $(BUILD_TARGETS) 30 | # all: $(BUILD_TARGETS) 31 | # $(BUILD_TARGETS): 32 | # $(MAKE) -f $(TOP_DIR)/stdapp.mk -C $(patsubst build-%,lib/%,$@) \ 33 | # -I $(TOP_DIR) ERL_DEPS_DIR=$(TOP_DIR)/build/$(@:build-%=%) \ 34 | # build 35 | # 36 | # Run "make build-foo" to build only the application foo. Add similar rules 37 | # for other targets like tests-foo, docs-foo, clean-foo, etc. Any specific 38 | # APPLICATION.mk files are expected to be in $(TOP_DIR)/apps/. If you don't 39 | # pass ERL_DEPS_DIR, the .d files will be placed in the ebin directory of the 40 | # app. Note that the $(MAKE) call runs from the app subdirectory, so it's 41 | # best to use absolute paths based on TOP_DIR for the parameters. 42 | # 43 | # * define STDAPP_NO_GIT_TAG if you don't want to compute git tags 44 | # * define STDAPP_FORCE_GIT_TAG_VSN if you want to always use git tags as vsn 45 | # * define STDAPP_NO_VSN_MK if you want to ignore any vsn.mk files 46 | # * define STDAPP_VSN_ADD_GIT_HASH if you want to add a git hash suffix 47 | # to the vsn (unless the vsn is equal to the git tag) 48 | # 49 | # Copyright (C) 2014 Klarna AB 50 | # 51 | # Permission is hereby granted, free of charge, to any person obtaining a 52 | # copy of this software and associated documentation files (the "Software"), 53 | # to deal in the Software without restriction, including without limitation 54 | # the rights to use, copy, modify, merge, publish, distribute, sublicense, 55 | # and/or sell copies of the Software, and to permit persons to whom the 56 | # Software is furnished to do so, subject to the following conditions: 57 | # 58 | # The above copyright notice and this permission notice shall be included in 59 | # all copies or substantial portions of the Software. 60 | # 61 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 62 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 63 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 64 | # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 65 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 66 | # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 67 | # DEALINGS IN THE SOFTWARE. 68 | 69 | # this ensures that no include file accidentally overrides the default rule 70 | .PHONY: all build tests clean docs distclean realclean 71 | all: build 72 | 73 | # read global configuration file, if it exists - not required to make clean 74 | # (use the -I flag with Make to specify the directory for this file) 75 | -include config.mk 76 | 77 | # variable defaults 78 | VPATH ?= 79 | LIB_DIR ?= $(abspath ..) 80 | ERL ?= erl 81 | ERL_NOSHELL ?= erl -noshell +A0 82 | ERLC ?= erlc 83 | ESCRIPT ?= escript 84 | EBIN_DIR ?= ebin 85 | SRC_DIR ?= src 86 | INCLUDE_DIR ?= include 87 | PRIV_DIR ?= priv 88 | DOC_DIR ?= doc 89 | TEST_DIR ?= test 90 | BIN_DIR ?= bin 91 | ERL_DEPS_DIR ?= $(EBIN_DIR) 92 | PROGRESS ?= @echo -n '.' 93 | GAWK ?= gawk 94 | DEFAULT_VSN ?= 0.1 95 | 96 | # figure out the application name, unless APPLICATION is already set 97 | # (first check for src/*.app.src, then ebin/*.app, otherwise use the dirname) 98 | ifndef APPLICATION 99 | appsrc = $(wildcard $(SRC_DIR)/*.app.src) 100 | ifneq ($(appsrc),) 101 | APPLICATION := $(patsubst $(SRC_DIR)/%.app.src,%,$(appsrc)) 102 | else 103 | appfile = $(wildcard $(EBIN_DIR)/*.app) 104 | ifneq ($(appfile),) 105 | APPLICATION := $(patsubst $(EBIN_DIR)/%.app,%,$(appfile)) 106 | else 107 | APPLICATION := $(notdir $(CURDIR)) 108 | endif 109 | endif 110 | endif 111 | export APPLICATION 112 | APP_SRC_FILE ?= $(SRC_DIR)/$(APPLICATION).app.src 113 | APP_FILE ?= $(EBIN_DIR)/$(APPLICATION).app 114 | APPLICATION_NAME_MACRO ?= APPLICATION 115 | 116 | # ensure that all applications under lib are available to erlc when building 117 | ERL_LIBS ?= $(LIB_DIR) 118 | export ERL_LIBS 119 | 120 | # generic Erlang sources and targets 121 | YRL_SOURCES := $(wildcard $(SRC_DIR)/*.yrl $(SRC_DIR)/*/*.yrl \ 122 | $(SRC_DIR)/*/*/*.yrl) 123 | ERL_SOURCES := $(wildcard $(SRC_DIR)/*.erl $(SRC_DIR)/*/*.erl \ 124 | $(SRC_DIR)/*/*/*.erl) 125 | ERL_TEST_SOURCES := $(wildcard $(TEST_DIR)/*.erl $(TEST_DIR)/*/*.erl) 126 | 127 | # read any vsn.mk for backwards compatibility with many existing applications 128 | # NOTE: if you use vsn.mk, then add a .app file dependency like the following: 129 | # 130 | # $(APP_FILE): vsn.mk 131 | # 132 | ifndef STDAPP_NO_VSN_MK 133 | -include ./vsn.mk 134 | ifndef VSN 135 | # some apps define the _VSN varable instead 136 | vsnvar=$(shell echo $(APPLICATION) | tr a-z A-Z)_VSN 137 | ifdef $(vsnvar) 138 | VSN := $($(vsnvar)) 139 | endif 140 | endif 141 | endif 142 | 143 | ifdef STDAPP_NO_GIT_TAG 144 | GIT_TAG := 145 | else 146 | ifdef STDAPP_VSN_ADD_GIT_HASH 147 | longdesc=--long 148 | endif 149 | GIT_TAG := $(shell git describe --tags --always $(longdesc)) 150 | ifdef STDAPP_FORCE_GIT_TAG_VSN 151 | VSN := $(GIT_TAG) 152 | endif 153 | endif 154 | 155 | # if VSN not yet defined, get nonempty vsn from any existing .app.src or 156 | # .app file, use git tag, if any, or default (note that sed regexp matching 157 | # is greedy, so the rightmost {vsn, "..."} in the input will be selected) 158 | ifndef VSN 159 | VSN := $(shell echo '{vsn,"$(DEFAULT_VSN)"}' `cat $(APP_FILE) 2> /dev/null` '{vsn,"$(GIT_TAG)"}' `cat $(APP_SRC_FILE) 2> /dev/null` | sed -n 's/.*{[[:space:]]*vsn[[:space:]]*,[[:space:]]*"\([^"][^"]*\)".*/\1/p') 160 | endif 161 | 162 | ifdef STDAPP_VSN_ADD_GIT_HASH 163 | ifneq ($(VSN),$(GIT_TAG)) 164 | VSN := $(VSN)-g$(shell git rev-parse --short HEAD) 165 | endif 166 | endif 167 | 168 | # read any application-specific definitions and rules 169 | -include ./app.mk 170 | 171 | # read any system-specific definitions and rules for the application 172 | # (use the -I flag with Make to specify the directory for these files) 173 | -include apps/$(APPLICATION).mk 174 | 175 | # ensure sane default values if not already defined at this point 176 | ERLC_FLAGS ?= +debug_info +warn_obsolete_guard +warn_export_all 177 | YRL_FLAGS ?= 178 | EDOC_OPTS ?= {def,{version,"$(VSN)"}},todo,no_packages 179 | 180 | # automatically add the include directory to erlc options (the src directory 181 | # is added so that modules under test/ can be compiled using the same rule) 182 | ERLC_FLAGS += -I $(INCLUDE_DIR) -I $(SRC_DIR) -D$(APPLICATION_NAME_MACRO)="$(APPLICATION)" 183 | 184 | # computed targets 185 | YRL_OBJECTS := $(YRL_SOURCES:%.yrl=%.erl) 186 | BO_OBJECTS := $(BO_SOURCES:%.erl.in=%.erl) 187 | ERL_SOURCES += $(YRL_OBJECTS) $(BO_OBJECTS) 188 | ERL_OBJECTS := $(addprefix $(EBIN_DIR)/, $(notdir $(ERL_SOURCES:%.erl=%.beam))) 189 | ERL_TEST_OBJECTS := $(addprefix $(EBIN_DIR)/, $(notdir $(ERL_TEST_SOURCES:%.erl=%.beam))) 190 | ERL_DEPS=$(ERL_OBJECTS:$(EBIN_DIR)/%.beam=$(ERL_DEPS_DIR)/%.d) 191 | ERL_TEST_DEPS=$(ERL_TEST_OBJECTS:$(EBIN_DIR)/%.beam=$(ERL_DEPS_DIR)/%.d) 192 | MODULES := $(sort $(ERL_OBJECTS:$(EBIN_DIR)/%.beam=%)) 193 | 194 | # comma-separated list of single-quoted module names 195 | # (the comma/space variables are needed to work around Make's argument parsing) 196 | comma := , 197 | space := 198 | space += 199 | MODULES_LIST := $(subst $(space),$(comma)$(space),$(patsubst %,'%',$(MODULES))) 200 | 201 | # add the list of directories containing source files to VPATH (note that 202 | # $(sort) removes duplicates; also ensure that at least $(ERL_DEPS_DIR) and 203 | # $(SRC_DIR) are always present in the VPATH even if there are no sources) 204 | VPATH := $(sort $(VPATH) $(dir $(ERL_SOURCES) $(ERL_TEST_SOURCES)) \ 205 | $(SRC_DIR)/ $(ERL_DEPS_DIR)/) 206 | 207 | # 208 | # Targets 209 | # 210 | 211 | .SUFFIXES: .erl .beam .yrl .d .app .app.src _bo.erl _bo.erl.in 212 | 213 | .PRECIOUS: $(BO_OBJECTS) $(YRL_OBJECTS) 214 | 215 | # read the .d file corresponding to each .erl file, UNLESS making clean! 216 | ifeq (,$(findstring clean,$(MAKECMDGOALS))) 217 | -include $(ERL_DEPS) 218 | # only read the .d file for test modules if actually building tests 219 | ifeq (tests,$(filter tests,$(MAKECMDGOALS))) 220 | -include $(ERL_TEST_DEPS) 221 | endif 222 | endif 223 | 224 | build: $(ERL_OBJECTS) $(APP_FILE) 225 | @$(ERL_NOSHELL) -eval 'erlang:halt(case file:consult("$(APP_FILE)") of {ok,_}->0; _->1 end)' || { echo '*** error: $(APP_FILE) is not readable'; exit 1; } 226 | 227 | tests: $(ERL_TEST_OBJECTS) 228 | 229 | realclean: distclean clean-docs 230 | rm -f $(APP_FILE) 231 | 232 | distclean: clean 233 | rm -f $(ERL_DEPS) $(ERL_TEST_DEPS) 234 | 235 | clean: clean-tests 236 | rm -f $(ERL_OBJECTS) $(BO_OBJECTS) $(YRL_OBJECTS) 237 | 238 | .PHONY: clean-tests 239 | clean-tests: 240 | rm -f $(ERL_TEST_OBJECTS) 241 | 242 | docs: $(DOC_DIR)/edoc-info 243 | 244 | # Note that we must run edoc from the src directory due to existing @docfile 245 | # "../doc/*.edoc" directives in all the *_bo.erl.in files 246 | BO_FIELDS_EDOC := $(patsubst $(SRC_DIR)/%_bo.erl, $(DOC_DIR)/%_bo_fields.edoc, $(BO_OBJECTS)) 247 | .PRECIOUS: $(BO_FIELDS_EDOC) 248 | $(DOC_DIR)/edoc-info: $(ERL_SOURCES) $(wildcard $(DOC_DIR)/*.edoc) \ 249 | $(BO_FIELDS_EDOC) 250 | $(PROGRESS) 251 | cd $(SRC_DIR) && $(ERL_NOSHELL) -eval 'edoc:application($(APPLICATION), "..", [$(EDOC_OPTS)]), init:stop().' 252 | 253 | .PHONY: clean-docs 254 | clean-docs: 255 | rm -f $(DOC_DIR)/edoc-info $(DOC_DIR)/*.html $(DOC_DIR)/stylesheet.css $(DOC_DIR)/erlang.png $(BO_FIELDS_EDOC) 256 | 257 | # this replaces existing {vsn, ...} and {modules, ...} in the app.src file 258 | # (note the special sed loop here to merge any multi-line modules declarations) 259 | $(APP_FILE): $(APP_SRC_FILE) | $(EBIN_DIR) 260 | $(PROGRESS) 261 | sed -e 's/{[[:space:]]*vsn[[:space:]]*,[[:space:]]*\({[^}]*}\)\?[^}]*}/{vsn, "$(VSN)"}/' \ 262 | -e ':x;/{[[:space:]]*modules[[:space:]]*,[[:space:]]*[^}]*$$/{N;b x}' \ 263 | -e "s/{[[:space:]]*modules[[:space:]]*,[[:space:]]*[^}]*}/{modules, [$(MODULES_LIST)]}/" \ 264 | $< > $@ 265 | 266 | # create a new .app.src file, or just clone the .app file if it already exists 267 | # (note: overwriting is easier than a multi-line conditional in a recipe) 268 | $(APP_SRC_FILE): 269 | $(PROGRESS) 270 | mkdir -p $(dir $@) 271 | echo > $@ '{application,$(APPLICATION),' 272 | echo >> $@ ' [{description,"The $(APPLICATION) application"},' 273 | echo >> $@ ' {vsn,"$(VSN)"},' 274 | echo >> $@ '% {mod,{$(APPLICATION)_app,[]}},' 275 | echo >> $@ ' {modules,[]},' 276 | echo >> $@ ' {registered, []},' 277 | echo >> $@ ' {applications,[kernel,stdlib]},' 278 | echo >> $@ ' {env, []}' 279 | echo >> $@ ' ]}.' 280 | if [ -f $(APP_FILE) ]; then sed -e 's/{[[:space:]]*vsn[[:space:]]*,[[:space:]]*[^}]*}/{vsn, "$(VSN)"}/' $(APP_FILE) > $(@); fi 281 | 282 | # ensuring that target directories exist; use order-only prerequisites for this 283 | $(sort $(EBIN_DIR) $(ERL_DEPS_DIR)): 284 | mkdir -p $@ 285 | 286 | # 287 | # Pattern rules 288 | # 289 | 290 | $(EBIN_DIR)/%.beam: %.erl | $(EBIN_DIR) 291 | $(PROGRESS) 292 | $(ERLC) $(ERLC_FLAGS) -o $(EBIN_DIR) $< 293 | 294 | %.erl: %.yrl 295 | $(PROGRESS) 296 | $(ERLC) $(YRL_FLAGS) -o $(dir $@) $< 297 | 298 | # automatically generated dependencies for header files and local behaviours 299 | # (there is no point in generating dependencies for behaviours in other 300 | # applications, since we cannot cause them to be built from the current app) 301 | # NOTE: currently doesn't find behaviour/transform modules in subdirs of src 302 | $(ERL_DEPS_DIR)/%.d: %.erl | $(ERL_DEPS_DIR) 303 | $(PROGRESS) 304 | $(ERLC) $(ERLC_FLAGS) -o $(ERL_DEPS_DIR) -MP -MG -MF $@ -MT "$(EBIN_DIR)/$*.beam $@" $< 305 | $(GAWK) '/^[ \t]*-(behaviou?r\(|compile\({parse_transform,)/ {match($$0, /-(behaviou?r\([ \t]*([^) \t]+)|compile\({parse_transform,[ \t]*([^} \t]+))/, a); m = (a[2] a[3]); if (m != "" && (getline x < ("$(SRC_DIR)/" m ".erl")) >= 0 || (getline x < ("$(TEST_DIR)/" m ".erl")) >= 0) print "\n$(EBIN_DIR)/$*.beam: $(EBIN_DIR)/" m ".beam"}' < $< >> $@ 306 | -------------------------------------------------------------------------------- /test/pg_perf.erl: -------------------------------------------------------------------------------- 1 | %%% 2 | %%% Copyright (c) 2014-2015 Klarna AB 3 | %%% 4 | %%% This file is provided to you under the Apache License, 5 | %%% Version 2.0 (the "License"); you may not use this file 6 | %%% except in compliance with the License. You may obtain 7 | %%% a copy of the License at 8 | %%% 9 | %%% http://www.apache.org/licenses/LICENSE-2.0 10 | %%% 11 | %%% Unless required by applicable law or agreed to in writing, 12 | %%% software distributed under the License is distributed on an 13 | %%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | %%% KIND, either express or implied. See the License for the 15 | %%% specific language governing permissions and limitations 16 | %%% under the License. 17 | 18 | -module(pg_perf). 19 | 20 | -include_lib("stdlib/include/ms_transform.hrl"). 21 | 22 | -export([setup/0, 23 | fill_table/2, fill_table/4, 24 | iterate_table/2, iterate_table/4, 25 | search_table/2, search_table/4, 26 | cleanup/0, 27 | test_avg/4]). 28 | 29 | -export([seed/0, seed/3]). 30 | 31 | -export([write/2, write/3, 32 | read/2, read/3, 33 | search/2, search/3]). 34 | 35 | -record(x, {name, value, fat}). 36 | 37 | write(Tab, Rec) -> 38 | write(transaction, Tab, Rec). 39 | 40 | write(Type, Tab, Rec) -> 41 | Fun = fun() -> 42 | mnesia:write(Tab, Rec, write) 43 | end, 44 | mnesia:activity(Type, Fun). 45 | 46 | read(Tab, Key) -> 47 | read(transaction, Tab, Key). 48 | 49 | read(Type, Tab, Key) -> 50 | Fun = fun() -> 51 | mnesia:read(Tab, Key, read) 52 | end, 53 | mnesia:activity(Type, Fun). 54 | 55 | search(Tab, MS) -> 56 | search(transaction, Tab, MS). 57 | 58 | search(Type, Tab, MS) -> 59 | F = fun() -> mnesia:select(Tab, MS) end, 60 | mnesia:activity(Type, F). 61 | 62 | setup_mnesia() -> 63 | stopped = mnesia:stop(), 64 | ok = mnesia:delete_schema([node()]), 65 | ok = mnesia:create_schema([node()], [{backend_types, 66 | [{pg_copies, 67 | mnesia_pg}, 68 | {leveldb_copies, 69 | klarna_leveldb_backend}]}]), 70 | ok = mnesia:start(). 71 | 72 | setup() -> 73 | setup_mnesia(), 74 | {atomic,ok} = mnesia:create_table(d, [{disc_copies, [node()]}, 75 | {attributes, record_info(fields, x)}, 76 | {record_name, x}]), 77 | {atomic,ok} = mnesia:create_table(do, [{disc_only_copies, [node()]}, 78 | {attributes, record_info(fields, x)}, 79 | {record_name, x}]), 80 | {atomic,ok} = mnesia:create_table(pgb, [{pg_copies, [node()]}, 81 | {attributes, record_info(fields, x)}, 82 | {record_name, x}]), 83 | {atomic,ok} = mnesia:create_table(ldb, [{leveldb_copies, [node()]}, 84 | {attributes, record_info(fields, x)}, 85 | {record_name, x}]), 86 | ok = mnesia:wait_for_tables([d,do,pgb,ldb], 30000), 87 | ok. 88 | 89 | cleanup() -> 90 | mnesia:delete_table(d), 91 | mnesia:delete_table(do), 92 | mnesia:delete_table(pgb), 93 | mnesia:delete_table(ldb), 94 | ok. 95 | 96 | get_key(X) -> 97 | {X, key}. 98 | 99 | get_value(X) -> 100 | {value, X, 101 | "valuable stuff " ++ integer_to_list(X), 102 | list_to_binary("valuable stuff " ++ integer_to_list(X)), 103 | math:pow(X,3)}. 104 | 105 | get_fat(0, Acc) -> 106 | Acc; 107 | get_fat(N, Acc) -> 108 | get_fat(N-1, <>). 109 | 110 | fill_table(Tab, N) -> 111 | fill_table(transaction, seq, Tab, N). 112 | 113 | fill_table(Type, Keyspace, Tab, N) -> 114 | %% Fat = get_fat(10*1014, <<>>), 115 | Fat = <<>>, 116 | foreach(fun(X) -> 117 | Name = get_key(X), 118 | Val = get_value(X), 119 | Bin = <>, 120 | Rec = #x{name=Name, value=Val, fat=Bin}, 121 | write(Type, Tab, Rec) 122 | end, Keyspace, N). 123 | 124 | iterate_table(Tab, N) -> 125 | iterate_table(transaction, seq, Tab, N). 126 | 127 | iterate_table(Type, Keyspace, Tab, N) -> 128 | fold(fun(X,Acc) -> 129 | Name = get_key(X), 130 | [#x{name = Name}] = read(Type, Tab, Name), 131 | Acc+1 132 | end, 0, Keyspace, N). 133 | 134 | search_table(Tab, N) -> 135 | search_table(transaction, seq, Tab, N). 136 | 137 | search_table(Type, Keyspace, Tab, N) -> 138 | fold(fun(X, Acc) -> 139 | Name = get_key(X), 140 | Val = get_value(X), 141 | [Val] = 142 | search(Type, Tab, [{{x,Name,'$1', '_'},[],['$1']}]), 143 | Acc + 1 144 | end, 0, Keyspace, N). 145 | 146 | test_avg(M, F, A, N) when N > 0 -> 147 | L = test_loop(M, F, A, N, []), 148 | Length = length(L), 149 | Min = lists:min(L), 150 | Max = lists:max(L), 151 | Med = lists:nth(round((Length / 2)), lists:sort(L)), 152 | Avg = round(lists:foldl(fun(X, Sum) -> X + Sum end, 0, L) / Length), 153 | io:format("Range: ~b - ~b mics~n" 154 | "Median: ~b mics~n" 155 | "Average: ~b mics~n", 156 | [Min, Max, Med, Avg]), 157 | Med. 158 | 159 | test_loop(_M, _F, _A, 0, List) -> 160 | List; 161 | test_loop(M, F, A, N, List) -> 162 | {T, _Result} = timer:tc(M, F, A), 163 | test_loop(M, F, A, N - 1, [T|List]). 164 | 165 | foreach(F, seq, N) -> 166 | lists:foreach(F, lists:seq(1, N)); 167 | foreach(F, random, N) -> 168 | foreach(F, {random, seed(), N}, N); 169 | foreach(F, {random, Seed}, N) -> 170 | foreach(F, {random, Seed, N}, N); 171 | foreach(F, {random, Seed, Range}, N) -> 172 | foreach_f(F, fun(_) -> random_s(Range, Seed) end, N); 173 | foreach(F, KS, N) when is_function(KS, 1) -> 174 | foreach_f(F, KS, N). 175 | 176 | foreach_f(_, _, 0) -> ok; 177 | foreach_f(F, KS, N) -> 178 | case KS(N) of 179 | done -> ok; 180 | {Key, KS1} -> 181 | F(Key), 182 | foreach_f(F, KS1, N-1) 183 | end. 184 | 185 | fold(F, Acc, seq, N) -> 186 | lists:foldl(F, Acc, lists:seq(1, N)); 187 | fold(F, Acc, random, N) -> 188 | fold(F, Acc, {random, seed(), N}, N); 189 | fold(F, Acc, {random, Seed}, N) -> 190 | fold(F, Acc, {random, Seed, N}, N); 191 | fold(F, Acc, {random, Seed, Range}, N) -> 192 | fold_f(F, Acc, fun(_) -> random_s(Range, Seed) end, N); 193 | fold(F, Acc, KS, N) when is_function(KS, 1) -> 194 | fold_f(F, Acc, KS, N). 195 | 196 | fold_f(_, Acc, _, 0) -> Acc; 197 | fold_f(F, Acc, KS, N) -> 198 | case KS(N) of 199 | done -> Acc; 200 | {Key, KS1} -> 201 | fold_f(F, F(Key, Acc), KS1, N-1) 202 | end. 203 | 204 | random_s(Range, Seed) -> 205 | {Key, NewSeed} = random:uniform_s(Range, Seed), 206 | {Key, fun(_) -> random_s(Range, NewSeed) end}. 207 | 208 | seed() -> 209 | {M, S, U} = os:timestamp(), 210 | seed(M, S, U). 211 | 212 | %% Create seed without side-effects (or at least repairing the unavoidable one) 213 | seed(A,B,C) -> 214 | OldSeed = random:seed(A, B, C), 215 | NewSeed = get(random_seed), 216 | put(random_seed, OldSeed), 217 | NewSeed. 218 | 219 | -------------------------------------------------------------------------------- /test/pg_proper_semantics.erl: -------------------------------------------------------------------------------- 1 | %%% 2 | %%% Copyright (c) 2014-2015 Klarna AB 3 | %%% 4 | %%% This file is provided to you under the Apache License, 5 | %%% Version 2.0 (the "License"); you may not use this file 6 | %%% except in compliance with the License. You may obtain 7 | %%% a copy of the License at 8 | %%% 9 | %%% http://www.apache.org/licenses/LICENSE-2.0 10 | %%% 11 | %%% Unless required by applicable law or agreed to in writing, 12 | %%% software distributed under the License is distributed on an 13 | %%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | %%% KIND, either express or implied. See the License for the 15 | %%% specific language governing permissions and limitations 16 | %%% under the License. 17 | 18 | %% This module uses the proper_statem pattern to generate random 19 | %% sequences of commands, mixing dirty and transaction operations 20 | %% (including dirty ops from within transactions). Each sequence is run 21 | %% against a disc_only_copies table and a pg_copies table, after 22 | %% which the result of each operation in the sequence is compared between 23 | %% the two runs. The postcondition is that every command in every sequence 24 | %% should yield the same value against both backends. 25 | 26 | -module(pg_proper_semantics). 27 | 28 | -export([test/0, 29 | test/1, 30 | setup/0, 31 | setup_mnesia/0, 32 | prop_seq/0]). 33 | 34 | %% statem callbacks 35 | -export([initial_state/0, 36 | command/1, 37 | precondition/2, 38 | postcondition/3, 39 | next_state/3]). 40 | 41 | %% command callbacks 42 | -export([activity/2]). 43 | 44 | -include_lib("proper/include/proper.hrl"). 45 | 46 | -record(st, {}). 47 | -define(KEYS, [a,b,c]). 48 | 49 | test() -> 50 | test(100). 51 | 52 | test(N) -> 53 | setup_mnesia(), 54 | true = proper:quickcheck(?MODULE:prop_seq(), N), 55 | ok. 56 | 57 | prop_seq() -> 58 | ?FORALL(Cmds, proper_statem:commands(?MODULE), 59 | begin 60 | setup(), 61 | {H, S, Res} = 62 | proper_statem:run_commands(?MODULE, Cmds), 63 | cleanup(), 64 | ?WHENFAIL( 65 | io:fwrite("History: ~w~n" 66 | "State : ~w~n" 67 | "Result : ~w~n", [H, S, Res]), 68 | proper:aggregate( 69 | proper_statem:command_names(Cmds), Res =:= ok)) 70 | end). 71 | 72 | setup_mnesia() -> 73 | stopped = mnesia:stop(), 74 | ok = mnesia:delete_schema([node()]), 75 | ok = mnesia:create_schema([node()], [{backend_types, 76 | [{pg_copies, 77 | mnesia_pg}]}]), 78 | ok = mnesia:start(). 79 | 80 | setup() -> 81 | {atomic,ok} = mnesia:create_table(d, [{disc_copies, [node()]}, 82 | {record_name, x}]), 83 | {atomic,ok} = mnesia:create_table(pg, [{pg_copies, [node()]}, 84 | {record_name, x}]), 85 | ok = mnesia:wait_for_tables([d,pg], 30000), 86 | ok. 87 | 88 | cleanup() -> 89 | {atomic, ok} = mnesia:delete_table(d), 90 | {atomic, ok} = mnesia:delete_table(pg), 91 | ok. 92 | 93 | initial_state() -> 94 | #st{}. 95 | 96 | command(#st{}) -> 97 | ?LET(Type, type(), 98 | {call, ?MODULE, activity, [Type, sequence()]}). 99 | 100 | type() -> 101 | proper_types:oneof([async_dirty, transaction]). 102 | 103 | precondition(_, _) -> 104 | true. 105 | 106 | postcondition(_, {call,?MODULE,activity,_}, {A, B}) -> 107 | A == B; 108 | postcondition(_, _, _) -> 109 | false. 110 | 111 | next_state(St, _, _) -> 112 | St. 113 | 114 | sequence() -> 115 | proper_types:list(db_cmd()). 116 | 117 | db_cmd() -> 118 | ?LET(Type, type(), 119 | proper_types:oneof([{Type, read, key()}, 120 | {Type, write, key(), value()}, 121 | {Type, delete, key()}])). 122 | 123 | key() -> 124 | proper_types:oneof([a,b,c]). 125 | 126 | value() -> 127 | proper_types:oneof([1,2,3]). 128 | 129 | activity(Type, Seq) -> 130 | {mnesia:activity(Type, fun() -> 131 | apply_seq(Type, d, Seq) 132 | end), 133 | mnesia:activity(Type, fun() -> 134 | apply_seq(Type, pg, Seq) 135 | end)}. 136 | 137 | apply_seq(Type, Tab, Seq) -> 138 | apply_seq(Type, Tab, Seq, []). 139 | 140 | apply_seq(transaction=X, Tab, [H|T], Acc) -> 141 | Res = case H of 142 | {X,read, K} -> mnesia:read(Tab, K, read); 143 | {_,read, K} -> mnesia:dirty_read(Tab,K); 144 | {X,write,K,V} -> mnesia:write(Tab, {x, K, V}, write); 145 | {_,write,K,V} -> mnesia:dirty_write(Tab, {x,K,V}); 146 | {X,delete,K} -> mnesia:delete(Tab, K, write); 147 | {_,delete,K} -> mnesia:dirty_delete(Tab,K) 148 | end, 149 | apply_seq(X, Tab, T, [Res|Acc]); 150 | apply_seq(X, Tab, [H|T], Acc) -> 151 | Res = case H of 152 | {_,read, K} -> mnesia:read(Tab, K, read); 153 | {_,write,K,V} -> mnesia:write(Tab, {x, K, V}, write); 154 | {_,delete,K} -> mnesia:delete(Tab, K, write) 155 | end, 156 | apply_seq(X, Tab, T, [Res|Acc]); 157 | apply_seq(_, _, [], Acc) -> 158 | lists:reverse(Acc). 159 | -------------------------------------------------------------------------------- /test/pg_test.erl: -------------------------------------------------------------------------------- 1 | %%% 2 | %%% Copyright (c) 2014-2015 Klarna AB 3 | %%% 4 | %%% This file is provided to you under the Apache License, 5 | %%% Version 2.0 (the "License"); you may not use this file 6 | %%% except in compliance with the License. You may obtain 7 | %%% a copy of the License at 8 | %%% 9 | %%% http://www.apache.org/licenses/LICENSE-2.0 10 | %%% 11 | %%% Unless required by applicable law or agreed to in writing, 12 | %%% software distributed under the License is distributed on an 13 | %%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | %%% KIND, either express or implied. See the License for the 15 | %%% specific language governing permissions and limitations 16 | %%% under the License. 17 | 18 | -module(pg_test). 19 | 20 | -include_lib("stdlib/include/ms_transform.hrl"). 21 | 22 | -export([t0/0, t1/0, t2/0, t3/0, t3b/0, t4/0, t5/0, dbg/1]). 23 | 24 | -record(x, {name, value}). 25 | 26 | dbg(on) -> 27 | dbg:start(), 28 | dbg:tracer(), 29 | dbg:tpl(mnesia, '_', []), 30 | %dbg:tpl(mnesia_locker, '_', []), 31 | dbg:tpl(mnesia_lib, '_', []), 32 | dbg:tpl(mnesia_pg, '_', []), 33 | dbg:tpl(pgsql, '_', []), 34 | dbg:p(all,c); 35 | dbg(off) -> 36 | dbg:stop_clear(). 37 | 38 | t0() -> 39 | setup_mnesia(), 40 | {atomic,ok} = mnesia:create_table(pg0, [{pg_copies, [node()]}, 41 | {attributes, record_info(fields, x)}, 42 | {record_name, x}]), 43 | mnesia:wait_for_tables([pg0], 30000). 44 | 45 | t1() -> 46 | Fun = fun() -> 47 | mnesia:write(pg0, {x, allan, 123}, write), 48 | mnesia:read(pg0, allan) 49 | end, 50 | mnesia:transaction(Fun). 51 | 52 | t2() -> 53 | mnesia:dirty_read(pg0, allan). 54 | 55 | t3() -> 56 | MS = ets:fun2ms(fun(X) -> X end), %select * 57 | F = fun() -> mnesia:select(pg0,MS) end, 58 | mnesia:transaction(F). 59 | 60 | t3b() -> 61 | MS = ets:fun2ms(fun(#x{name=X}) -> X end), 62 | F = fun() -> mnesia:select(pg0,MS) end, 63 | mnesia:transaction(F). 64 | 65 | t4() -> 66 | F = fun() -> mnesia:match_object(pg0, {x,'_','_'}, read) end, 67 | mnesia:transaction(F). 68 | 69 | t5() -> 70 | Fun = fun() -> 71 | mnesia:delete(pg0, allan, write) 72 | end, 73 | mnesia:transaction(Fun). 74 | 75 | setup_mnesia() -> 76 | stopped = mnesia:stop(), 77 | ok = mnesia:delete_schema([node()]), 78 | ok = mnesia:create_schema([node()], [{backend_types, 79 | [{pg_copies, 80 | mnesia_pg}]}]), 81 | ok = mnesia:start(). 82 | --------------------------------------------------------------------------------