├── LICENSE ├── README.md ├── include ├── kai.hrl └── kai_test.hrl ├── src ├── kai.app.src ├── kai.erl ├── kai_config.erl ├── kai_connection.erl ├── kai_coordinator.erl ├── kai_hash.erl ├── kai_log.erl ├── kai_membership.erl ├── kai_memcache.erl ├── kai_rpc.erl ├── kai_stat.erl ├── kai_store.erl ├── kai_store_dets.erl ├── kai_store_ets.erl ├── kai_sup.erl ├── kai_sync.erl ├── kai_tcp_server.erl ├── kai_tcp_server_acceptor.erl ├── kai_tcp_server_monitor.erl ├── kai_tcp_server_sup.erl ├── kai_version.erl └── vclock.erl └── test ├── kai.config ├── kai.coverspec ├── kai_config_SUITE.erl ├── kai_connection_SUITE.erl ├── kai_coordinator_SUITE.erl ├── kai_hash_SUITE.erl ├── kai_log_SUITE.erl ├── kai_membership_SUITE.erl ├── kai_memcache_SUITE.erl ├── kai_rpc_SUITE.erl ├── kai_stat_SUITE.erl ├── kai_store_SUITE.erl ├── kai_sync_SUITE.erl ├── kai_tcp_server_SUITE.erl ├── kai_version_SUITE.erl └── vclock_SUITE.erl /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | KAI 2 | === 3 | 4 | Kai is a distributed key-value datastore, which is mainly inspired 5 | by Amazon's Dynamo. It brings high scalability and availability 6 | to your Web sites. You can manage variety of contents with Kai, 7 | as if Amazon's Dynamo stores shopping carts, catalogs, session states, 8 | and so forth. Currently, Kai is deployed in a commercial service, 9 | http://home.goo.ne.jp, which is a Japanese social networking service of more 10 | than 10 million users. 11 | 12 | Features 13 | -------- 14 | 15 | * 16 nodes 16 | * 2 GB/node, on memory 17 | * 1,000 qps for whole system 18 | * < 300ms for requests of 99% 19 | * Fairly balancing loads 20 | * Consitent hashing, nodes, buckets 21 | * Secondary Indexes 22 | * Storage Backends: ETS, DETS 23 | * Quorum coordinator 24 | * Sync data one by another 25 | * Gossip-based protocol membership 26 | * Memcache Interface 27 | 28 | Goal 29 | ------ 30 | 31 | * 256 nodes 32 | * 32 GB/node 33 | * 100,000 qps for whole system 34 | * Physical placement 35 | * Merkle Tree sync 36 | * Chord or Kademlia membeship 37 | 38 | Standalone Mode 39 | --------------- 40 | 41 | For practice, we begin with a stand-alone server, not clustered system. 42 | The stand-alone Kai is not attractive, because it has no reliability 43 | (no replication). The behavior is, however, quite similar with that 44 | of well-known memcached, and so this can be a good starting point. 45 | Before running Kai, stop a memcached if runnnig, because Kai uses 46 | the same port number (11211) by default. Set parameters "n", "r", 47 | and "w" to 1 in the configuration file "sys.config"; other parameters 48 | remain in default (Details for the parameters are described in Configuration). 49 | 50 | [{kai, [ 51 | {logfile, "kai.log"}, 52 | {hostname, "localhost"}, 53 | {rpc_port, 11011}, 54 | {rpc_max_processes, 30}, 55 | {memcache_port, 11211}, 56 | {memcache_max_processes, 10}, 57 | {max_connections, 32}, 58 | {n, 1}, 59 | {r, 1}, 60 | {w, 1}, 61 | {number_of_buckets, 1024}, 62 | {number_of_virtual_nodes, 128}, 63 | {store, ets}, 64 | {dets_dir, "/path/to/dir"}, 65 | {number_of_tables, 256}]}]. 66 | 67 | Cluster Mode 68 | ------------ 69 | 70 | This section describes how to run multiple Kai nodes as a clustered system. 71 | While we will run all nodes on a single physical machine for convenience, 72 | the cluster can be consisted of different machines just by changing IP 73 | addresses and port numbers in the following example. In this example, 74 | we run four distinct nodes on a single physical machine. Copy the 75 | configuration file for them. They are named "kai1.config" to "kai4.config"; 76 | a node configured by kai1.config is called Node1, hereafter. 77 | To replicate data, set the quorum parameter "n" to the degree of replication. 78 | In this example, we set "n" to 3 in the configuration file "kai.config". 79 | Other quorum parameters "r" and "w" is set in according to the quorum conditions. 80 | 81 | 82 | | Name | Value | 83 | |------|-------| 84 | | n | 3 | 85 | | r | 2 | 86 | | w | 2 | 87 | 88 | Hosts config: 89 | 90 | | Node1 | Value | 91 | |---------------|-------| 92 | | rpc_port | 11011 | 93 | | memcache_port | 11211 | 94 | 95 | | Node2 | Value | 96 | |---------------|-------| 97 | | rpc_port | 11012 | 98 | | memcache_port | 11212 | 99 | 100 | | Node3 | Value | 101 | |---------------|-------| 102 | | rpc_port | 11013 | 103 | | memcache_port | 11213 | 104 | 105 | | Node4 | Value | 106 | |---------------|-------| 107 | | rpc_port | 11014 | 108 | | memcache_port | 11214 | 109 | 110 | Staring Node1: 111 | 112 | % erl -pa ebin -config kai1 113 | > application:start(kai). 114 | 115 | To make a cluster, start Node2 and add it to the cluster (that includes only 116 | Node1 currently) by informing of their neighbors with function "kai_rpc:check_node/2". 117 | 118 | % erl -pa ebin -config kai2 119 | > application:start(kai). 120 | > kai_rpc:check_node({{127,0,0,1}, 11012}, {{127,0,0,1}, 11011}). 121 | > kai_rpc:check_node({{127,0,0,1}, 11011}, {{127,0,0,1}, 11012}). 122 | 123 | We give a brief explanation this process. A Kai node maintains a node list 124 | including members of the cluster it belongs to. In the initial state, each 125 | node list includes only the node itself. By issuing function "kai_rpc:check_node/2", 126 | a node of the first argument retrieves a node list from another node of 127 | the second argument, and merge the list with that of itself. By calling 128 | this function twice, a new node can be added to the cluster; inform a 129 | new node of any cluster node by the first call, and vice versa (wait 130 | for finishing data synchronization if some data have already been 131 | stored in the cluster). 132 | 133 | In this example, Node2 (127.0.0.1:11012) gets a node list of 134 | Node1 (127.0.0.1:11011) by the first "kai_rpc:check_node/2"; Node2 135 | knows Node1. The second RPC informs Node1 of Node2. 136 | 137 | Here, make sure Node1 and Node2 know each other. 138 | 139 | > kai_rpc:node_list({{127,0,0,1}, 11011}). 140 | {node_list,[{{127,0,0,1},11011}, 141 | {{127,0,0,1},11012}]} 142 | 143 | > kai_rpc:node_list({{127,0,0,1}, 11012}). 144 | {node_list,[{{127,0,0,1},11011}, 145 | {{127,0,0,1},11012}]} 146 | 147 | In the same manner, add Node3 to the cluster. 148 | 149 | % erl -pa ebin -config kai3 150 | > application:start(kai). 151 | > kai_rpc:check_node({{127,0,0,1}, 11013}, {{127,0,0,1}, 11011}). 152 | > kai_rpc:check_node({{127,0,0,1}, 11011}, {{127,0,0,1}, 11013}). 153 | 154 | By the third "kai_rpc:check_node/2", Node3 gets a node list of Node1, 155 | which includes Node2 as well as Node1. The fourth "kai_rpc:check_node/2" 156 | informs Node1 of Node3. Now, while Node1 and Node3 know all the three 157 | nodes, Node2 does not know Node3 yet. Node2, however, gets to know 158 | Node3 eventually, because each node periodically exchanges its node 159 | list to a node randomly chosen among the node list. Every node, 160 | finally, knows each other, and makes the cluster of the three nodes. 161 | Finally, add Node4 to the cluster. 162 | 163 | % erl -pa ebin -config kai4 164 | > application:start(kai). 165 | > kai_rpc:check_node({{127,0,0,1}, 11014}, {{127,0,0,1}, 11011}). 166 | > kai_rpc:check_node({{127,0,0,1}, 11011}, {{127,0,0,1}, 11014}). 167 | 168 | We make sure whether each node knows all by function "kai_rpc:node_list/1". 169 | If the cluster is correctly made up, the output includes the four nodes like this. 170 | 171 | > kai_rpc:node_list({{127,0,0,1}, 11011}). 172 | {node_list,[{{127,0,0,1},11011}, 173 | {{127,0,0,1},11012}, 174 | {{127,0,0,1},11013}, 175 | {{127,0,0,1},11014}]} 176 | 177 | To remove a node, no procedure is needed. This implies that no 178 | action is needed when a node is accidentally going down. However, 179 | it's better to remove a node one by one; don't remove more than or 180 | equal to N nodes before finishing data synchronization. 181 | 182 | Statistics 183 | ---------- 184 | 185 | Statistics for the Kai node can be retrieved by issuing stats 186 | command of memcache APIs. The parameters without "kai_" or "erlang_" prefix 187 | are in common with memcache APIs. Some of Kai related parameters are 188 | explained in Configuration in detail. 189 | 190 | | Name | Description | 191 | |------------------|-------------------------------------------------------------------------| 192 | | uptime | Number of seconds this server has been running. | 193 | | time | Current UNIX time according to the server. | 194 | | version | Version string of the Kai node. | 195 | | bytes | Current number of bytes used by the Kai node to store items. | 196 | | curr_items | Current number of items stored by the Kai node. | 197 | | curr_connections | Number of open connections. | 198 | | cmd_get | Successful retrieval requests processed by the Kai node. | 199 | | cmd_set | Successful storage requests processed by the Kai node. | 200 | | bytes_read | Total number of bytes transferred from the Kai node. | 201 | | bytes_write | Total number of bytes transferred to the Kai node. | 202 | | kai_node | Socket address for internal RPC in the Kai node. | 203 | | kai_quorum | Parameters for the quorum protocol; i.e. N:R:W. | 204 | | kai_buckets | Number of buckets in the Kai cluster. | 205 | | kai_vnodes | Number of virtual nodes in the Kai node. | 206 | | kai_store | Storage type used by the Kai node; ets or dets. | 207 | | kai_curr_nodes | Membership in the current cluster. | 208 | | kai_unreconciled | Cumulative number of retrieval requests with confliction. | 209 | | erlang_procs | Number of Erlang processes. | 210 | | erlang_version | Version string of Erlang. | 211 | 212 | Some statistics (bytes, curr_items, cmd_get, cmd_set, bytes_read, and bytes_write) can be 213 | drawn by Cacti, which is a complete network graphing solution designed to harness the power 214 | of RRDTool's data storage and graphing functionality. 215 | 216 | Credits 217 | ------- 218 | 219 | * Takeshi Inoue 220 | * Maxim Sokhatsky 221 | 222 | OM A HUM 223 | -------------------------------------------------------------------------------- /include/kai.hrl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -record(data, { 14 | key, bucket, last_modified, vector_clocks, checksum, flags, value 15 | }). 16 | 17 | -record(tcp_server_option, { 18 | listen = [{active, false}, binary, {packet, line}, {reuseaddr, true}], 19 | port = 11211, 20 | max_processes = 8, 21 | max_restarts = 3, 22 | time = 60, 23 | shutdown = 2000, 24 | accept_timeout = infinity, 25 | accept_error_sleep_time = 3000, 26 | recv_length = 0, 27 | recv_timeout = infinity 28 | }). 29 | 30 | -define(error (Data), kai_log:log(error, self(), ?FILE, ?LINE, Data)). 31 | -define(warning(Data), kai_log:log(warning, self(), ?FILE, ?LINE, Data)). 32 | -define(info (Data), kai_log:log(info, self(), ?FILE, ?LINE, Data)). 33 | 34 | %-define(debug(Data), kai_log:log(debug, self(), ?FILE, ?LINE, Data)). 35 | -define(debug(_Data), ok). 36 | 37 | -define(TIMEOUT, 5000). 38 | -define(TIMER, 1000). 39 | -------------------------------------------------------------------------------- /include/kai_test.hrl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -define(NODE1, {{127,0,0,1}, 11011}). 14 | -define(NODE2, {{127,0,0,1}, 11012}). 15 | -define(NODE3, {{127,0,0,1}, 11013}). 16 | -define(NODE4, {{127,0,0,1}, 11014}). 17 | 18 | -define(INFO, [{number_of_virtual_nodes, 2}]). 19 | 20 | -define(assert(BoolExpr), 21 | ((fun() -> 22 | case (BoolExpr) of 23 | true -> ok; 24 | __V -> .erlang:error({assertion_failed, 25 | [{module, ?MODULE}, 26 | {line, ?LINE}, 27 | {expression, (??BoolExpr)}, 28 | {expected, true}, 29 | {value, case __V of 30 | false -> __V; 31 | _ -> {not_a_boolean,__V} 32 | end}]}) 33 | end 34 | end)())). 35 | 36 | -define(assertNot(BoolExpr), ?assert(not (BoolExpr))). 37 | 38 | -define(assertEqual(Expect, Expr), 39 | ((fun(__X) -> 40 | case (Expr) of 41 | __X -> ok; 42 | __V -> .erlang:error({assertEqual_failed, 43 | [{module, ?MODULE}, 44 | {line, ?LINE}, 45 | {expression, (??Expr)}, 46 | {expected, __X}, 47 | {value, __V}]}) 48 | end 49 | end)(Expect))). 50 | -------------------------------------------------------------------------------- /src/kai.app.src: -------------------------------------------------------------------------------- 1 | {application, kai, 2 | [{description, "KAI Dynamo Inspired DHT"}, 3 | {vsn, "1"}, 4 | {modules, [ 5 | kai, kai_sup, kai_memcache, kai_rpc, kai_tcp_server, kai_coordinator, 6 | kai_membership, kai_sync, kai_connection, kai_version, kai_stat, 7 | kai_store, kai_hash, kai_log, kai_config 8 | ]}, 9 | {registered, [ 10 | kai_sup, kai_membership, kai_sync, kai_connection, kai_version, 11 | kai_stat, kai_store, kai_hash, kai_log, kai_config 12 | ]}, 13 | {applications, [kernel, stdlib]}, 14 | {mod, {kai, []}}, 15 | {start_phases, []}, 16 | {env, [ 17 | {logfile, "kai.log"}, 18 | {hostname, "localhost"}, 19 | {rpc_port, 11011}, 20 | {rpc_max_processes, 30}, 21 | {memcache_port, 11211}, 22 | {memcache_max_processes, 10}, 23 | {max_connections, 32}, 24 | {n, 3}, 25 | {r, 2}, 26 | {w, 2}, 27 | {number_of_buckets, 10}, 28 | {number_of_virtual_nodes, 128}, 29 | {store, dets}, 30 | {dets_dir, "kai"}, 31 | {number_of_tables, 1024} 32 | ]} 33 | ]}. 34 | -------------------------------------------------------------------------------- /src/kai.erl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -module(kai). 14 | -behaviour(application). 15 | 16 | -export([start/2, stop/1]). 17 | -export([start/0]). 18 | 19 | config([], Acc) -> 20 | Acc; 21 | config([Key|Rest], Acc) -> 22 | case application:get_env(kai, Key) of 23 | undefined -> config(Rest, Acc); 24 | {ok, Value} -> config(Rest, [{Key, Value}|Acc]) 25 | end. 26 | 27 | start(_Type, _Args) -> 28 | filelib:ensure_dir("kai/"), 29 | Args = config([ 30 | logfile, hostname, 31 | rpc_port, rpc_max_processes, 32 | memcache_port, memcache_max_processes, 33 | max_connections, 34 | n, r, w, 35 | number_of_buckets, number_of_virtual_nodes, 36 | store, dets_dir, number_of_tables 37 | ], []), 38 | kai_sup:start_link(Args). 39 | 40 | stop(_State) -> 41 | ok. 42 | 43 | start() -> 44 | application:start(kai). 45 | -------------------------------------------------------------------------------- /src/kai_config.erl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -module(kai_config). 14 | -behaviour(gen_server). 15 | 16 | -export([start_link/1, stop/0]). 17 | -export([get/1, node_info/0]). 18 | -export([ 19 | init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, 20 | code_change/3 21 | ]). 22 | 23 | -include("kai.hrl"). 24 | 25 | -define(SERVER, ?MODULE). 26 | 27 | start_link(Args) -> 28 | gen_server:start_link({local, ?SERVER}, ?MODULE, Args, _Opts = []). 29 | 30 | init(Args) -> 31 | ets:new(config, [set, private, named_table]), 32 | 33 | lists:foreach( 34 | fun({Key, Value}) -> ets:insert(config, {Key, Value}) end, 35 | Args 36 | ), 37 | 38 | Hostname = 39 | case proplists:get_value(hostname, Args) of 40 | undefined -> {ok, H} = inet:gethostname(), H; 41 | H -> H 42 | end, 43 | {ok, Address} = inet:getaddr(Hostname, inet), 44 | Port = proplists:get_value(rpc_port, Args), 45 | ets:insert(config, {node, {Address, Port}}), 46 | 47 | NumberOfBuckets = proplists:get_value(number_of_buckets, Args), 48 | Exponent = round( math:log(NumberOfBuckets) / math:log(2) ), 49 | ets:insert(config, {number_of_buckets, trunc( math:pow(2, Exponent) )}), 50 | 51 | {ok, []}. 52 | 53 | terminate(_Reason, _State) -> 54 | ets:delete(config), 55 | ok. 56 | 57 | do_get(Key) -> 58 | case ets:lookup(config, Key) of 59 | [{Key, Value}|_] -> Value; 60 | _ -> undefined 61 | end. 62 | 63 | do_get([], ListOfValues) -> 64 | lists:reverse(ListOfValues); 65 | do_get([Key|Rest], ListOfValues) -> 66 | do_get(Rest, [do_get(Key)|ListOfValues]). 67 | 68 | get(ListOfKeys, State) when is_list(ListOfKeys)-> 69 | {reply, do_get(ListOfKeys, []), State}; 70 | get(Key, State) -> 71 | {reply, do_get(Key), State}. 72 | 73 | node_info(State) -> 74 | [LocalNode, NumberOfVirtualNode] = 75 | do_get([node, number_of_virtual_nodes], []), 76 | Info = [{number_of_virtual_nodes, NumberOfVirtualNode}], 77 | {reply, {node_info, LocalNode, Info}, State}. 78 | 79 | handle_call(stop, _From, State) -> 80 | {stop, normal, stopped, State}; 81 | handle_call({get, Key}, _From, State) -> 82 | get(Key, State); 83 | handle_call(node_info, _From, State) -> 84 | node_info(State). 85 | handle_cast(_Msg, State) -> 86 | {noreply, State}. 87 | handle_info(_Info, State) -> 88 | {noreply, State}. 89 | code_change(_OldVsn, State, _Extra) -> 90 | {ok, State}. 91 | 92 | stop() -> 93 | gen_server:call(?SERVER, stop). 94 | get(Key) -> 95 | gen_server:call(?SERVER, {get, Key}). 96 | node_info() -> 97 | gen_server:call(?SERVER, node_info). 98 | -------------------------------------------------------------------------------- /src/kai_connection.erl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -module(kai_connection). 14 | -behaviour(gen_server). 15 | 16 | -export([start_link/0, stop/0]). 17 | -export([lease/2, lease/3, return/1, close/1, connections/0]). 18 | -export([ 19 | init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, 20 | code_change/3 21 | ]). 22 | 23 | -include("kai.hrl"). 24 | 25 | -define(SERVER, ?MODULE). 26 | 27 | -record(connection, {node, available, socket}). 28 | 29 | start_link() -> 30 | gen_server:start_link({local, ?SERVER}, ?MODULE, [], _Opts = []). 31 | 32 | init(_Args) -> 33 | {ok, []}. 34 | 35 | terminate(_Reason, _State) -> 36 | ok. 37 | 38 | do_lease({Address, Port} = Node, Pid, Opts, [], Acc) -> 39 | case gen_tcp:connect( 40 | Address, Port, 41 | [binary, {active, true}, {packet, 4}, {reuseaddr, true}], 42 | ?TIMEOUT 43 | ) of 44 | {ok, Socket} -> 45 | case gen_tcp:controlling_process(Socket, Pid) of 46 | ok -> 47 | ok = inet:setopts(Socket, Opts), 48 | Connection = #connection{ 49 | node = Node, 50 | available = false, 51 | socket = Socket 52 | }, 53 | Connections = [Connection|lists:reverse(Acc)], % LRU 54 | {ok, Socket, Connections}; 55 | {error, Reason} -> 56 | {error, Reason, Acc} 57 | end; 58 | {error, Reason} -> 59 | {error, Reason, Acc} 60 | end; 61 | do_lease(Node, Pid, Opts, [#connection{node=Node, available=true, socket=Socket}|Rest], Acc) -> 62 | case gen_tcp:controlling_process(Socket, Pid) of 63 | ok -> 64 | ok = inet:setopts(Socket, Opts), 65 | Connection = #connection{ 66 | node = Node, 67 | available = false, 68 | socket = Socket 69 | }, 70 | Connections = [Connection|lists:reverse(Acc)] ++ Rest, % LRU 71 | flush(Socket), 72 | {ok, Socket, Connections}; 73 | {error, Reason} -> 74 | Connections = Acc ++ Rest, 75 | {error, Reason, Connections} 76 | end; 77 | do_lease(Node, Pid, Opts, [C|Rest], Acc) -> 78 | do_lease(Node, Pid, Opts, Rest, [C|Acc]). 79 | 80 | lease(Node, Pid, Opts, Connections) -> 81 | case do_lease(Node, Pid, Opts, Connections, []) of 82 | {ok, Socket, Connections2} -> 83 | Connections3 = lru(Connections2), 84 | {reply, {ok, Socket}, Connections3}; 85 | {error, Reason, Connections2} -> 86 | ?warning(io_lib:format("lease(~p, ~p) ~p", 87 | [Node, Pid, {error, Reason}])), 88 | {reply, {error, Reason}, Connections2} 89 | end. 90 | 91 | flush(Socket) -> 92 | receive 93 | {tcp, Socket, _Bin} -> flush(Socket) 94 | after 0 -> ok 95 | end. 96 | 97 | lru(0, Rest, Acc) -> 98 | lists:reverse(Rest) ++ Acc; 99 | lru(_N, [], Acc) -> 100 | Acc; 101 | lru(N, [#connection{node=_, available=true, socket=Socket}|Rest], Acc) -> 102 | gen_tcp:close(Socket), 103 | lru(N-1, Rest, Acc); 104 | lru(N, [#connection{node=_, available=false, socket=_} = C|Rest], Acc) -> 105 | lru(N, Rest, [C|Acc]). 106 | 107 | lru(Connections) -> 108 | MaxConnections = kai_config:get(max_connections), 109 | Len = length(Connections), 110 | if 111 | Len > MaxConnections -> 112 | lru(Len - MaxConnections, lists:reverse(Connections), []); 113 | true -> 114 | Connections 115 | end. 116 | 117 | do_return(_Socket, [], Acc) -> 118 | {error, enoent, Acc}; 119 | do_return(Socket, [#connection{node=Node, available=_, socket=Socket}|Rest], Acc) -> 120 | Connection = #connection{ 121 | node = Node, 122 | available = true, 123 | socket = Socket 124 | }, 125 | Connections = [Connection|lists:reverse(Acc)] ++ Rest, % LRU 126 | {ok, Connections}; 127 | do_return(Socket, [C|Rest], Acc) -> 128 | do_return(Socket, Rest, [C|Acc]). 129 | 130 | return(Socket, Connections) -> 131 | case do_return(Socket, Connections, []) of 132 | {ok, Connections2} -> 133 | Connections3 = lru(Connections2), 134 | {reply, ok, Connections3}; 135 | {error, Reason, Connections2} -> 136 | ?warning(io_lib:format("return(~p) ~p", 137 | [Socket, {error, Reason}])), 138 | {reply, {error, Reason}, Connections2} 139 | end. 140 | 141 | 142 | do_close(_Socket, [], Acc) -> 143 | {error, enoent, Acc}; 144 | do_close(Socket, [#connection{node=_, available=_, socket=Socket}|Rest], Acc) -> 145 | gen_tcp:close(Socket), 146 | {ok, lists:reverse(Acc) ++ Rest}; 147 | do_close(Socket, [C|Rest], Acc) -> 148 | do_close(Socket, Rest, [C|Acc]). 149 | 150 | close(Socket, Connections) -> 151 | case do_close(Socket, Connections, []) of 152 | {ok, Connections2} -> 153 | {reply, ok, Connections2}; 154 | {error, Reason, Connections2} -> 155 | {reply, {error, Reason}, Connections2} 156 | end. 157 | 158 | connections(Connections) -> 159 | {reply, {ok, Connections}, Connections}. 160 | 161 | handle_call(stop, _From, State) -> 162 | {stop, normal, stopped, State}; 163 | handle_call({lease, Node, Pid, Opts}, _From, State) -> 164 | lease(Node, Pid, Opts, State); 165 | handle_call({return, Socket}, _From, State) -> 166 | return(Socket, State); 167 | handle_call({close, Socket}, _From, State) -> 168 | close(Socket, State); 169 | handle_call(connections, _From, State) -> 170 | connections(State). 171 | handle_cast(_Msg, State) -> 172 | {noreply, State}. 173 | handle_info(_Info, State) -> 174 | {noreply, State}. 175 | code_change(_OldVsn, State, _Extra) -> 176 | {ok, State}. 177 | 178 | stop() -> 179 | gen_server:call(?SERVER, stop). 180 | lease(Node, Pid) when is_pid(Pid)-> 181 | gen_server:call(?SERVER, {lease, Node, Pid, []}). 182 | lease(Node, Pid, Opts) when is_pid(Pid)-> 183 | gen_server:call(?SERVER, {lease, Node, Pid, Opts}). 184 | return(Socket) when is_port(Socket) -> 185 | case reset_controlling_process(Socket) of 186 | ok -> 187 | gen_server:call(?SERVER, {return, Socket}); 188 | {error, Reason} -> 189 | ?warning(io_lib:format("return(~p) ~p", [Socket, {error, Reason}])), 190 | gen_server:call(?SERVER, {close, Socket}), 191 | {error, Reason} 192 | end. 193 | close(Socket) -> 194 | gen_server:call(?SERVER, {close, Socket}). 195 | connections() -> 196 | gen_server:call(?SERVER, connections). 197 | 198 | reset_controlling_process(Socket) -> 199 | case whereis(?SERVER) of 200 | Pid when is_pid(Pid) -> gen_tcp:controlling_process(Socket, Pid); 201 | _ -> {error, esrch} 202 | end. 203 | -------------------------------------------------------------------------------- /src/kai_coordinator.erl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -module(kai_coordinator). 14 | 15 | -export([route/1]). 16 | -export([start_route/3, map_in_get/4, map_in_put/4, map_in_delete/4]). 17 | 18 | -include("kai.hrl"). 19 | 20 | -define(SERVER, ?MODULE). 21 | 22 | dispatch({Type, Data} = _Request) -> 23 | case Type of 24 | get -> coordinate_get(Data); 25 | put -> coordinate_put(Data); 26 | delete -> coordinate_delete(Data); 27 | _Other -> {error, ebadrpc} 28 | end. 29 | 30 | do_route(_Request, []) -> 31 | {error, ebusy}; 32 | do_route({_Type, Data} = Request, [Node|RestNodes]) -> 33 | % TODO: introduce TTL, in order to avoid infinite loop 34 | case kai_rpc:route(Node, Request) of 35 | {error, Reason} -> 36 | ?warning(io_lib:format("do_route(~p, ~p): ~p", 37 | [Data#data.key, Node, {error, Reason}])), 38 | do_route(Request, RestNodes); 39 | Results -> 40 | Results 41 | end. 42 | 43 | start_route({_Type, Data} = Request, Pid, Ref) -> 44 | LocalNode = kai_config:get(node), 45 | {nodes, Nodes} = kai_hash:find_nodes(Data#data.key), 46 | Results = 47 | case lists:member(LocalNode, Nodes) of 48 | true -> dispatch(Request); 49 | _ -> do_route(Request, Nodes) 50 | end, 51 | Pid ! {Ref, Results}. 52 | 53 | route({_Type, Data} = Request) -> 54 | Ref = make_ref(), 55 | % Though don't know the reason, application exits abnormally if it doesn't 56 | % spawn the process 57 | spawn(?MODULE, start_route, [Request, self(), Ref]), 58 | receive 59 | {Ref, Result} -> Result 60 | after ?TIMEOUT -> 61 | ?warning(io_lib:format("route(~p): timeout", [Data#data.key])), 62 | [] 63 | end. 64 | 65 | coordinate_get(Data) -> 66 | {bucket, Bucket} = kai_hash:find_bucket(Data#data.key), 67 | {nodes, Nodes } = kai_hash:find_nodes(Bucket), 68 | Data2 = Data#data{bucket=Bucket}, 69 | Ref = make_ref(), 70 | lists:foreach( 71 | fun(Node) -> spawn(?MODULE, map_in_get, [Node, Data2, Ref, self()]) end, % Don't link 72 | Nodes 73 | ), 74 | [N, R] = kai_config:get([n, r]), 75 | case gather_in_get(Ref, N, R, []) of 76 | ListOfData when is_list(ListOfData) -> 77 | %% TODO: write back recent if multiple versions are found and they can be resolved 78 | InternalNum = sets:size( 79 | sets:from_list( 80 | lists:map(fun(E) -> E#data.vector_clocks end, 81 | ListOfData))), 82 | ReconciledList = kai_version:order(ListOfData), 83 | if 84 | InternalNum > 1 -> 85 | kai_stat:incr_unreconciled_get( 86 | {InternalNum, length(ReconciledList) =:= 1}); 87 | true -> ok 88 | end, 89 | ReconciledList; 90 | _NoData -> 91 | undefined 92 | end. 93 | 94 | map_in_get(Node, Data, Ref, Pid) -> 95 | case kai_rpc:get(Node, Data) of 96 | {error, Reason} -> 97 | % kai_membership:check_node(Node), 98 | Pid ! {Ref, {error, Reason}}; 99 | Other -> 100 | Pid ! {Ref, Other} 101 | end. 102 | 103 | gather_in_get(_Ref, _N, 0, Results) -> 104 | Results; 105 | gather_in_get(_Ref, 0, _R, _Results) -> 106 | {error, enodata}; 107 | gather_in_get(Ref, N, R, Results) -> 108 | receive 109 | {Ref, Data} when is_record(Data, data) -> 110 | gather_in_get(Ref, N-1, R-1, [Data|Results]); 111 | {Ref, undefined} -> 112 | gather_in_get(Ref, N-1, R-1, Results); 113 | {Ref, _Other} -> 114 | gather_in_get(Ref, N-1, R, Results) 115 | after ?TIMEOUT -> 116 | ?warning("gather_in_get/4: timeout"), 117 | Results 118 | end. 119 | 120 | coordinate_put(Data) -> 121 | Key = Data#data.key, 122 | Flags = Data#data.flags, 123 | Value = Data#data.value, 124 | {bucket, Bucket} = kai_hash:find_bucket(Key), 125 | {nodes, Nodes } = kai_hash:find_nodes(Bucket), 126 | Ref = make_ref(), 127 | Data1 = 128 | case kai_store:get(Data#data{bucket=Bucket}) of 129 | PreviousData when is_record(PreviousData, data) -> 130 | PreviousData; 131 | undefined -> 132 | #data{key=Key, vector_clocks=vclock:fresh()} 133 | end, 134 | {ok, Data2} = kai_version:update(Data1), 135 | Data3 = Data2#data{ 136 | bucket = Bucket, 137 | checksum = erlang:md5(Value), 138 | flags = Flags, 139 | value = Value 140 | }, 141 | lists:foreach( 142 | fun(Node) -> spawn(?MODULE, map_in_put, [Node, Data3, Ref, self()]) end, 143 | Nodes 144 | ), 145 | [N, W] = kai_config:get([n, w]), 146 | gather_in_put(Ref, N, W). 147 | 148 | map_in_put(Node, Data, Ref, Pid) -> 149 | case kai_rpc:put(Node, Data) of 150 | {error, Reason} -> 151 | % kai_membership:check_node(Node), 152 | Pid ! {Ref, {error, Reason}}; 153 | Other -> 154 | Pid ! {Ref, Other} 155 | end. 156 | 157 | gather_in_put(_Ref, _N, 0) -> 158 | ok; 159 | gather_in_put(_Ref, 0, _W) -> 160 | {error, ebusy}; 161 | gather_in_put(Ref, N, W) -> 162 | receive 163 | {Ref, ok} -> gather_in_put(Ref, N-1, W-1); 164 | {Ref, _Other} -> gather_in_put(Ref, N-1, W) 165 | after ?TIMEOUT -> 166 | ?warning("gather_in_put/3: timeout"), 167 | {error, etimedout} 168 | end. 169 | 170 | coordinate_delete(Data) -> 171 | {bucket, Bucket} = kai_hash:find_bucket(Data#data.key), 172 | {nodes, Nodes } = kai_hash:find_nodes(Bucket), 173 | Data2 = Data#data{bucket=Bucket}, 174 | Ref = make_ref(), 175 | lists:foreach( 176 | fun(Node) -> spawn(?MODULE, map_in_delete, [Node, Data2, Ref, self()]) end, 177 | Nodes 178 | ), 179 | [N, W] = kai_config:get([n, w]), 180 | gather_in_delete(Ref, N, W, []). 181 | 182 | map_in_delete(Node, Data, Ref, Pid) -> 183 | case kai_rpc:delete(Node, Data) of 184 | {error, Reason} -> 185 | % kai_membership:check_node(Node), 186 | Pid ! {Ref, {error, Reason}}; 187 | Other -> 188 | Pid ! {Ref, Other} 189 | end. 190 | 191 | gather_in_delete(_Ref, _N, 0, Results) -> 192 | case lists:member(ok, Results) of 193 | true -> ok; 194 | _ -> undefined 195 | end; 196 | gather_in_delete(_Ref, 0, _W, _Results) -> 197 | {error, ebusy}; 198 | gather_in_delete(Ref, N, W, Results) -> 199 | receive 200 | {Ref, ok} -> 201 | gather_in_delete(Ref, N-1, W-1, [ok|Results]); 202 | {Ref, undefined} -> 203 | gather_in_delete(Ref, N-1, W-1, [undefined|Results]); 204 | {Ref, _Other} -> 205 | gather_in_delete(Ref, N-1, W, Results) 206 | after ?TIMEOUT -> 207 | ?warning("gather_in_delete/4: timeout"), 208 | {error, etimedout} 209 | end. 210 | -------------------------------------------------------------------------------- /src/kai_hash.erl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -module(kai_hash). 14 | -behaviour(gen_server). 15 | -compile(export_all). 16 | -export([start_link/0, stop/0]). 17 | -export([ 18 | update_nodes/2, find_bucket/1, find_replica/1, find_nodes/1, 19 | choose_node_randomly/0, choose_bucket_randomly/0, 20 | node_info/1, node_info/0, node_list/0, virtual_node_list/0, 21 | bucket_list/0, buckets/0 22 | ]). 23 | -export([ 24 | init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, 25 | code_change/3 26 | ]). 27 | 28 | -include("kai.hrl"). 29 | 30 | -define(SERVER, ?MODULE). 31 | -define(HASH_LEN, 32). 32 | 33 | start_link() -> 34 | gen_server:start_link({local, ?SERVER}, ?MODULE, [], _Opts = []). 35 | 36 | init(_Args) -> 37 | ets:new(node_list, [set, private, named_table]), 38 | ets:new(virtual_node_list, [ordered_set, private, named_table]), 39 | ets:new(buckets, [set, public, named_table]), 40 | 41 | {node_info, LocalNode, Info} = kai_config:node_info(), 42 | update_nodes([{LocalNode, Info}], [], _State = []), 43 | 44 | {ok, _State = []}. 45 | 46 | terminate(_Reason, _State) -> 47 | ets:delete(node_list), 48 | ets:delete(virtual_node_list), 49 | ets:delete(buckets), 50 | ok. 51 | 52 | hash(Key) -> 53 | <> = erlang:md5(Key), 54 | HashedKey. 55 | hash({{N1,N2,N3,N4}, Port}, VirtualNode) -> 56 | <> = 57 | erlang:md5(<>), 58 | HashedKey. 59 | 60 | bucket_range(NumberOfBuckets) -> 61 | trunc( math:pow(2, ?HASH_LEN) / NumberOfBuckets ). 62 | 63 | search_bucket_nodes(_HashedKey, _N, 0, Nodes) -> 64 | {nodes, lists:reverse(Nodes)}; 65 | search_bucket_nodes(HashedKey, N, I, Nodes) -> 66 | HashedNode = 67 | case ets:next(virtual_node_list, HashedKey) of 68 | '$end_of_table' -> ets:first(virtual_node_list); 69 | Other -> Other 70 | end, 71 | [{_HashedNode, Node}|_] = ets:lookup(virtual_node_list, HashedNode), 72 | Nodes2 = 73 | case lists:member(Node, Nodes) of 74 | true -> Nodes; 75 | _ -> [Node|Nodes] 76 | end, 77 | case length(Nodes2) of 78 | N -> {nodes, lists:reverse(Nodes2)}; 79 | _ -> search_bucket_nodes(HashedNode, N, I-1, Nodes2) 80 | end. 81 | 82 | lists_index(_Elem, [], _I) -> 83 | undefined; 84 | lists_index(Elem, [Head|Rest], I) -> 85 | case Elem =:= Head of 86 | true -> I; 87 | _ -> lists_index(Elem, Rest, I+1) 88 | end. 89 | lists_index(Elem, List) -> 90 | lists_index(Elem, List, 1). 91 | 92 | update_buckets(-1 = _Bucket, _LocalNode, _BucketRange, _N, _MaxSearch, 93 | ReplacedBuckets) -> 94 | {replaced_buckets, ReplacedBuckets}; 95 | update_buckets(Bucket, LocalNode, BucketRange, N, MaxSearch, 96 | ReplacedBuckets) -> 97 | {nodes, NewNodes} = 98 | search_bucket_nodes(Bucket * BucketRange, N, MaxSearch, []), 99 | case ets:lookup(buckets, Bucket) of 100 | [{Bucket, NewNodes}] -> 101 | update_buckets(Bucket-1, LocalNode, BucketRange, N, MaxSearch, 102 | ReplacedBuckets); 103 | OldBucket -> 104 | ets:insert(buckets, {Bucket, NewNodes}), 105 | NewReplica = lists_index(LocalNode, NewNodes), 106 | OldReplica = 107 | case OldBucket of 108 | [{Bucket, OldNodes}] -> lists_index(LocalNode, OldNodes); 109 | [] -> undefined 110 | end, 111 | ReplacedBuckets2 = 112 | case {NewReplica, OldReplica} of 113 | {Replica, Replica} -> 114 | ReplacedBuckets; 115 | _ -> 116 | [{Bucket, NewReplica, OldReplica}|ReplacedBuckets] 117 | end, 118 | update_buckets(Bucket-1, LocalNode, BucketRange, N, MaxSearch, 119 | ReplacedBuckets2) 120 | end. 121 | 122 | update_buckets() -> 123 | [LocalNode, N, NumberOfBuckets] = 124 | kai_config:get([node, n, number_of_buckets]), 125 | BucketRange = bucket_range(NumberOfBuckets), 126 | NumberOfNodes = proplists:get_value(size, ets:info(node_list)), 127 | 128 | % Don't search other nodes to fill a bucket when NumberOfNodes is 1, since 129 | % they are never found. 130 | MaxSearch = 131 | case NumberOfNodes of 132 | 1 -> 1; 133 | _ -> proplists:get_value(size, ets:info(virtual_node_list)) 134 | end, 135 | 136 | update_buckets(NumberOfBuckets-1, LocalNode, BucketRange, N, MaxSearch, []). 137 | 138 | add_nodes([]) -> 139 | ok; 140 | add_nodes([{Node, Info}|Rest]) -> 141 | case ets:lookup(node_list, Node) of 142 | [{Node, _Info}|_] -> ok; 143 | [] -> 144 | ets:insert(node_list, {Node, Info}), 145 | NumberOfVirtualNodes = 146 | proplists:get_value(number_of_virtual_nodes, Info), 147 | lists:foreach( 148 | fun(VirtualNode) -> 149 | HashedKey = hash(Node, VirtualNode), 150 | ets:insert(virtual_node_list, {HashedKey, Node}) 151 | end, 152 | lists:seq(1, NumberOfVirtualNodes) 153 | ) 154 | end, 155 | add_nodes(Rest). 156 | 157 | remove_nodes([]) -> 158 | ok; 159 | remove_nodes([Node|Rest]) -> 160 | case ets:lookup(node_list, Node) of 161 | [{Node, Info}|_] -> 162 | ets:delete(node_list, Node), 163 | NumberOfVirtualNodes = 164 | proplists:get_value(number_of_virtual_nodes, Info), 165 | lists:foreach( 166 | fun(VirtualNode) -> 167 | HashedKey = hash(Node, VirtualNode), 168 | ets:delete(virtual_node_list, HashedKey) 169 | end, 170 | lists:seq(1, NumberOfVirtualNodes) 171 | ); 172 | [] -> ok 173 | end, 174 | remove_nodes(Rest). 175 | 176 | update_nodes(NodesToAdd, NodesToRemove, State) -> 177 | LocalNode = kai_config:get(node), 178 | Reply = 179 | case {NodesToAdd, NodesToRemove -- [LocalNode]} of 180 | {[], []} -> 181 | {replaced_buckets, []}; 182 | _ -> 183 | ?info({update, NodesToAdd, NodesToRemove}), 184 | add_nodes(NodesToAdd), 185 | remove_nodes(NodesToRemove), 186 | update_buckets() 187 | end, 188 | {reply, Reply, State}. 189 | 190 | do_find_bucket(Bucket, NumberOfBuckets) when is_integer(Bucket) -> 191 | Bucket rem NumberOfBuckets; 192 | do_find_bucket(Key, NumberOfBuckets) -> 193 | hash(Key) div bucket_range(NumberOfBuckets). 194 | 195 | find_bucket(KeyOrBucket, State) -> 196 | NumberOfBuckets = kai_config:get(number_of_buckets), 197 | {reply, {bucket, do_find_bucket(KeyOrBucket, NumberOfBuckets)}, State}. 198 | 199 | find_replica(KeyOrBucket, State) -> 200 | LocalNode = kai_config:get(node), 201 | {reply, {nodes, Nodes}, State2} = find_nodes(KeyOrBucket, State), 202 | Replica = lists_index(LocalNode, Nodes), 203 | {reply, {replica, Replica}, State2}. 204 | 205 | find_nodes(KeyOrBucket, State) -> 206 | NumberOfBuckets = kai_config:get(number_of_buckets), 207 | Bucket = do_find_bucket(KeyOrBucket, NumberOfBuckets), 208 | [{Bucket, Nodes}|_] = ets:lookup(buckets, Bucket), 209 | {reply, {nodes, Nodes}, State}. 210 | 211 | choose_node_randomly(State) -> 212 | {{N1,N2,N3,N4}, Port} = kai_config:get(node), 213 | Head = {'$1', '_'}, 214 | Cond = [{'=/=', '$1', {{{{N1,N2,N3,N4}}, Port}}}], % double tuple paranthesis 215 | Body = ['$1'], 216 | Nodes = ets:select(node_list, [{Head, Cond, Body}]), 217 | Len = length(Nodes), 218 | case Len of 219 | 0 -> {reply, undefined, State}; 220 | _ -> {reply, {node, lists:nth(random:uniform(Len), Nodes)}, State} 221 | end. 222 | 223 | inversed_buckets(_Node, -1 = _Bucket, Buckets) -> 224 | Buckets; 225 | inversed_buckets(Node, Bucket, Buckets) -> 226 | [{Bucket, Nodes}|_] = ets:lookup(buckets, Bucket), 227 | case lists:member(Node, Nodes) of 228 | true -> inversed_buckets(Node, Bucket-1, [Bucket|Buckets]); 229 | _ -> inversed_buckets(Node, Bucket-1, Buckets) 230 | end. 231 | 232 | inversed_buckets(Node) -> 233 | NumberOfBuckets = kai_config:get(number_of_buckets), 234 | inversed_buckets(Node, NumberOfBuckets-1, []). 235 | 236 | choose_bucket_randomly(State) -> 237 | LocalNode = kai_config:get(node), 238 | Buckets = inversed_buckets(LocalNode), 239 | Len = length(Buckets), 240 | case Len of 241 | 0 -> {reply, undefined, State}; 242 | _ -> {reply, {bucket, lists:nth(random:uniform(Len), Buckets)}, State} 243 | end. 244 | 245 | do_node_info(Node, State) -> 246 | Head = {Node, '$2'}, 247 | Cond = [], 248 | Body = ['$2'], 249 | [Info] = ets:select(node_list, [{Head, Cond, Body}]), 250 | {reply, {node_info, Node, Info}, State}. 251 | 252 | do_node_info(State) -> 253 | LocalNode = kai_config:get(node), 254 | do_node_info(LocalNode, State). 255 | 256 | node_list(State) -> 257 | NodeList = ets:tab2list(node_list), 258 | NodeList2 = lists:map(fun({Node, _Info}) -> Node end, NodeList), 259 | {reply, {node_list, NodeList2}, State}. 260 | 261 | virtual_node_list(State) -> 262 | VirtualNodeList = ets:tab2list(virtual_node_list), 263 | {reply, {virtual_node_list, VirtualNodeList}, State}. 264 | 265 | bucket_list(State) -> 266 | Buckets = ets:tab2list(buckets), 267 | {reply, {bucket_list, lists:sort(Buckets)}, State}. 268 | 269 | buckets(State) -> 270 | LocalNode = kai_config:get(node), 271 | Buckets = 272 | lists:filter( 273 | fun(B) -> lists:member(LocalNode, element(2, B)) end, 274 | ets:tab2list(buckets) 275 | ), 276 | Buckets2 = [element(1, B) || B <- Buckets], 277 | {reply, {buckets, lists:sort(Buckets2)}, State}. 278 | 279 | handle_call(stop, _From, State) -> 280 | {stop, normal, stopped, State}; 281 | handle_call({update_nodes, NodesToAdd, NodesToRemove}, _From, State) -> 282 | update_nodes(NodesToAdd, NodesToRemove, State); 283 | handle_call({find_bucket, KeyOrBucket}, _From, State) -> 284 | find_bucket(KeyOrBucket, State); 285 | handle_call({find_replica, KeyOrBucket}, _From, State) -> 286 | find_replica(KeyOrBucket, State); 287 | handle_call({find_nodes, KeyOrBucket}, _From, State) -> 288 | find_nodes(KeyOrBucket, State); 289 | handle_call(choose_node_randomly, _From, State) -> 290 | choose_node_randomly(State); 291 | handle_call(choose_bucket_randomly, _From, State) -> 292 | choose_bucket_randomly(State); 293 | handle_call({node_info, Node}, _From, State) -> 294 | do_node_info(Node, State); 295 | handle_call(node_info, _From, State) -> 296 | do_node_info(State); 297 | handle_call(node_list, _From, State) -> 298 | node_list(State); 299 | handle_call(virtual_node_list, _From, State) -> 300 | virtual_node_list(State); 301 | handle_call(bucket_list, _From, State) -> 302 | bucket_list(State); 303 | handle_call(buckets, _From, State) -> 304 | buckets(State). 305 | handle_cast(_Msg, State) -> 306 | {noreply, State}. 307 | handle_info(_Info, State) -> 308 | {noreply, State}. 309 | code_change(_OldVsn, State, _Extra) -> 310 | {ok, State}. 311 | 312 | stop() -> 313 | gen_server:call(?SERVER, stop). 314 | update_nodes(NodesToAdd, NodesToRemove) -> 315 | gen_server:call(?SERVER, {update_nodes, NodesToAdd, NodesToRemove}). 316 | find_bucket(KeyOrBucket) -> 317 | gen_server:call(?SERVER, {find_bucket, KeyOrBucket}). 318 | find_replica(KeyOrBucket) -> 319 | gen_server:call(?SERVER, {find_replica, KeyOrBucket}). 320 | find_nodes(KeyOrBucket) -> 321 | gen_server:call(?SERVER, {find_nodes, KeyOrBucket}). 322 | choose_node_randomly() -> 323 | gen_server:call(?SERVER, choose_node_randomly). 324 | choose_bucket_randomly() -> 325 | gen_server:call(?SERVER, choose_bucket_randomly). 326 | node_info() -> 327 | gen_server:call(?SERVER, node_info). 328 | node_info(Node) -> 329 | gen_server:call(?SERVER, {node_info, Node}). 330 | node_list() -> 331 | gen_server:call(?SERVER, node_list). 332 | virtual_node_list() -> 333 | gen_server:call(?SERVER, virtual_node_list). 334 | bucket_list() -> 335 | gen_server:call(?SERVER, bucket_list). 336 | buckets() -> 337 | gen_server:call(?SERVER, buckets). 338 | -------------------------------------------------------------------------------- /src/kai_log.erl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -module(kai_log). 14 | -behaviour(gen_server). 15 | 16 | -export([start_link/0, stop/0]). 17 | -export([log/5]). 18 | -export([ 19 | init/1, terminate/2, handle_cast/2, handle_call/3, handle_info/2, 20 | code_change/3 21 | ]). 22 | 23 | -include("kai.hrl"). 24 | 25 | -define(SERVER, ?MODULE). 26 | 27 | start_link() -> 28 | gen_server:start_link({local, ?SERVER}, ?MODULE, [], _Opts = []). 29 | 30 | init(_Args) -> 31 | case kai_config:get(logfile) of 32 | undefined -> 33 | {ok, []}; 34 | File -> 35 | case file:open(File, [write, append]) of 36 | {ok, Fd} -> {ok, [{fd, Fd}]}; 37 | Error -> Error 38 | end 39 | end. 40 | 41 | terminate(_Reason, State) -> 42 | case proplists:get_value(fd, State) of 43 | undefined -> ok; 44 | Fd -> file:close(Fd) 45 | end. 46 | 47 | log(Type, Pid, File, Line, Data, State) -> 48 | {{Year,Month,Day}, {Hour,Minute,Second}} = erlang:localtime(), 49 | {_MegaSec, _Sec, Usec} = now(), 50 | Data2 = 51 | if 52 | is_list(Data) -> lists:flatten(Data); 53 | true -> Data 54 | end, 55 | Buf = io_lib:format( 56 | "~4..0w-~2..0w-~2..0w ~2..0w:~2..0w:~2..0w.~6..0w [~s] (~p) ~s:~w: ~p\n", 57 | [Year, Month, Day, Hour, Minute, Second, Usec, Type, Pid, File, Line, Data2] 58 | ), 59 | case proplists:get_value(fd, State) of 60 | undefined -> io:format( "~s", [Buf]); 61 | Fd -> io:format(Fd, "~s", [Buf]) 62 | end. 63 | 64 | handle_call(stop, _From, State) -> 65 | {stop, normal, stopped, State}. 66 | handle_cast({log, Type, Pid, File, Line, Data}, State) -> 67 | log(Type, Pid, File, Line, Data, State), 68 | {noreply, State}. 69 | handle_info(_Info, State) -> 70 | {noreply, State}. 71 | code_change(_OldVsn, State, _Extra) -> 72 | {ok, State}. 73 | 74 | stop() -> 75 | gen_server:call(?SERVER, stop). 76 | log(Type, Pid, File, Line, Data) -> 77 | gen_server:cast(?SERVER, {log, Type, Pid, File, Line, Data}). 78 | -------------------------------------------------------------------------------- /src/kai_membership.erl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -module(kai_membership). 14 | -behaviour(gen_fsm). 15 | 16 | -export([start_link/0, stop/0]). 17 | -export([check_node/1]). 18 | -export([ 19 | init/1, ready/2, handle_event/3, handle_sync_event/4, handle_info/3, 20 | terminate/3, code_change/4 21 | ]). 22 | 23 | -include("kai.hrl"). 24 | 25 | -define(SERVER, ?MODULE). 26 | 27 | start_link() -> 28 | gen_fsm:start_link({local, ?SERVER}, ?MODULE, [], _Opts = []). 29 | 30 | init(_Args) -> 31 | {ok, ready, [], ?TIMER}. 32 | 33 | terminate(_Reason, _StateName, _StateData) -> 34 | ok. 35 | 36 | ping_nodes([], AvailableNodes, DownNodes) -> 37 | {AvailableNodes, DownNodes}; 38 | ping_nodes([Node|Nodes], AvailableNodes, DownNodes) -> 39 | case kai_rpc:node_info(Node) of 40 | {node_info, Node2, Info} -> 41 | ping_nodes(Nodes, [{Node2, Info}|AvailableNodes], DownNodes); 42 | {error, Reason} -> 43 | ?warning(io_lib:format("ping_nodes/3: ~p", [{error, Reason}])), 44 | ping_nodes(Nodes, AvailableNodes, [Node|DownNodes]) 45 | end. 46 | 47 | retrieve_node_list(Node) -> 48 | case kai_rpc:node_list(Node) of 49 | {node_list, RemoteNodeList} -> 50 | {node_list, LocalNodeList} = kai_hash:node_list(), 51 | NewNodes = RemoteNodeList -- LocalNodeList, 52 | OldNodes = LocalNodeList -- RemoteNodeList, 53 | Nodes = NewNodes ++ OldNodes, 54 | LocalNode = kai_config:get(node), 55 | ping_nodes(Nodes -- [LocalNode], [], []); 56 | {error, Reason} -> 57 | ?warning(io_lib:format("retrieve_node_list/1: ~p", [{error, Reason}])), 58 | {[], [Node]} 59 | end. 60 | 61 | sync_buckets([], _LocalNode) -> 62 | ok; 63 | sync_buckets([{Bucket, NewReplica, OldReplica}|ReplacedBuckets], LocalNode) -> 64 | case {NewReplica, OldReplica} of 65 | {NewReplica, undefined } -> kai_sync:update_bucket(Bucket); 66 | {undefined, OldReplica} -> kai_sync:delete_bucket(Bucket); 67 | _ -> nop 68 | end, 69 | sync_buckets(ReplacedBuckets, LocalNode). 70 | 71 | sync_buckets(ReplacedBuckets) -> 72 | LocalNode = kai_config:get(node), 73 | sync_buckets(ReplacedBuckets, LocalNode). 74 | 75 | do_check_node({Address, Port}) -> 76 | {AvailableNodes, DownNodes} = retrieve_node_list({Address, Port}), 77 | {replaced_buckets, ReplacedBuckets} = 78 | kai_hash:update_nodes(AvailableNodes, DownNodes), 79 | sync_buckets(ReplacedBuckets). 80 | 81 | ready({check_node, Node}, State) -> 82 | do_check_node(Node), 83 | {next_state, ready, State, ?TIMER}; 84 | ready(timeout, State) -> 85 | case kai_hash:choose_node_randomly() of 86 | {node, Node} -> do_check_node(Node); 87 | _ -> nop 88 | end, 89 | {next_state, ready, State, ?TIMER}. 90 | 91 | handle_event(stop, _StateName, StateData) -> 92 | {stop, normal, StateData}. 93 | handle_sync_event(_Event, _From, _StateName, StateData) -> 94 | {next_state, ready, StateData, ?TIMEOUT}. 95 | handle_info(_Info, _StateName, StateData) -> 96 | {next_state, ready, StateData, ?TIMEOUT}. 97 | code_change(_OldVsn, _StateName, StateData, _Extra) -> 98 | {ok, ready, StateData}. 99 | 100 | stop() -> 101 | gen_fsm:send_all_state_event(?SERVER, stop). 102 | check_node(Node) -> 103 | gen_fsm:send_event(?SERVER, {check_node, Node}). 104 | -------------------------------------------------------------------------------- /src/kai_memcache.erl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -module(kai_memcache). 14 | -behaviour(kai_tcp_server). 15 | 16 | -export([start_link/0, stop/0]). 17 | -export([init/1, handle_call/3]). 18 | 19 | -include("kai.hrl"). 20 | 21 | -define(MEMCACHE_TIMEOUT, ?TIMEOUT). 22 | 23 | start_link() -> 24 | kai_tcp_server:start_link( 25 | {local, ?MODULE}, 26 | ?MODULE, 27 | [], 28 | #tcp_server_option{ 29 | port = kai_config:get(memcache_port), 30 | max_processes = kai_config:get(memcache_max_processes) 31 | } 32 | ). 33 | 34 | stop() -> kai_tcp_server:stop(?MODULE). 35 | 36 | init(_Args) -> {ok, {}}. 37 | 38 | handle_call(Socket, Data, State) -> 39 | dispatch(Socket, string:tokens(binary_to_list(Data), " \r\n"), State). 40 | 41 | dispatch(_Socket, ["get", Key], State) -> 42 | do_get(Key, State, false); 43 | dispatch(_Socket, ["gets", Key], State) -> 44 | do_get(Key, State, true); 45 | 46 | dispatch(Socket, ["set", _Key, _Flags, "0", _Bytes] = Data, State) -> 47 | inet:setopts(Socket, [{packet, raw}]), 48 | Result = recv_set_data(Socket, Data, State), 49 | inet:setopts(Socket, [{packet, line}]), 50 | Result; 51 | 52 | dispatch(_Socket, ["set", _Key, _Flags, _Exptime, _Bytes], State) -> 53 | {reply, <<"CLIENT_ERROR Exptime must be zero.\r\n">>, State}; 54 | 55 | dispatch(_Socket, ["delete", Key], State) -> 56 | case kai_coordinator:route({delete, #data{key=Key}}) of 57 | ok -> {reply, <<"DELETED\r\n">>, State}; 58 | undefined -> {reply, <<"NOT_FOUND\r\n">>, State}; 59 | _Other -> 60 | send_error_and_close(State, "Failed to delete.") 61 | end; 62 | 63 | dispatch(_Socket, ["delete", Key, "0"], State) -> 64 | dispatch(_Socket, ["delete", Key], State); 65 | dispatch(_Socket, ["delete", _Key, _Time], State) -> 66 | {reply, <<"CLIENT_ERROR Time must be zero.\r\n">>, State}; 67 | 68 | dispatch(_Socket, ["delete", _Key, _Time, "noreply"], State) -> 69 | {reply, <<"CLIENT_ERROR noreply not supported.\r\n">>, State}; 70 | 71 | dispatch(_Socket, ["stats"], State) -> 72 | Response = 73 | lists:map( 74 | fun({Name, Value}) -> 75 | ["STAT " ++ atom_to_list(Name) ++ " " ++ Value ++ "\r\n"] 76 | end, 77 | kai_stat:all() 78 | ), 79 | {reply, [Response|"END\r\n"], State}; 80 | 81 | dispatch(_Socket, ["version"], State) -> 82 | Version = 83 | case application:get_key(kai, vsn) of 84 | {ok, V} -> V; 85 | _ -> "0" 86 | end, 87 | {reply, "VERSION " ++ Version ++ "\r\n", State}; 88 | 89 | dispatch(_Socket, ["quit"], _State) -> quit; 90 | 91 | dispatch(_Socket, _Unknown, State) -> 92 | {reply, <<"ERROR\r\n">>, State}. 93 | 94 | do_get(Key, State, WithCasUnique) -> 95 | case kai_coordinator:route({get, #data{key=Key}}) of 96 | Data when is_list(Data) -> 97 | {ok, CasUniqueInBinary} = kai_version:cas_unique(Data), 98 | Response = get_response(Data, WithCasUnique, CasUniqueInBinary), 99 | kai_stat:incr_cmd_get(), 100 | kai_stat:add_bytes_read(Data), 101 | {reply, [Response|"END\r\n"], State}; 102 | undefined -> 103 | {reply, <<"END\r\n">>, State}; 104 | _Other -> 105 | send_error_and_close("Failed to read.", State) 106 | end. 107 | 108 | get_response(Data, WithCasUnique, CasUnique) -> 109 | lists:map( 110 | fun(Elem) -> 111 | Key = Elem#data.key, 112 | Flags = Elem#data.flags, 113 | Value = Elem#data.value, 114 | [ 115 | io_lib:format("VALUE ~s ~s ~w", [Key, Flags, byte_size(Value)]), 116 | case WithCasUnique of 117 | true -> 118 | io_lib:format(" ~w", [cas_unique(CasUnique)]); 119 | _ -> [] 120 | end, 121 | "\r\n", Value, "\r\n"] 122 | end, Data). 123 | 124 | recv_set_data(Socket, ["set", Key, Flags, "0", Bytes], State) -> 125 | case gen_tcp:recv(Socket, list_to_integer(Bytes), ?MEMCACHE_TIMEOUT) of 126 | {ok, Value} -> 127 | gen_tcp:recv(Socket, 2, ?MEMCACHE_TIMEOUT), 128 | case kai_coordinator:route( 129 | {put, #data{key=Key, flags=Flags, value=Value}} 130 | ) of 131 | ok -> 132 | gen_tcp:send(Socket, <<"STORED\r\n">>), 133 | kai_stat:incr_cmd_set(), 134 | kai_stat:add_bytes_write(#data{value=Value}), 135 | {noreply, State}; 136 | _Other -> 137 | send_error_and_close("Failed to write.", State) 138 | end; 139 | _Other -> 140 | {noreply, State} 141 | end. 142 | 143 | send_error_and_close(Message, State) -> 144 | ?warning(io_lib:format("send_error_and_close/2: ~p", [Message])), 145 | {close, ["SERVER_ERROR ", Message, "\r\n"], State}. 146 | 147 | cas_unique(CasUniqueInBinary) -> 148 | <> = CasUniqueInBinary, 149 | HashedValue. 150 | -------------------------------------------------------------------------------- /src/kai_rpc.erl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -module(kai_rpc). 14 | -behaviour(kai_tcp_server). 15 | 16 | -export([start_link/0, stop/0]). 17 | -export([init/1, handle_call/3]). 18 | -export([ 19 | node_info/1, node_list/1, 20 | list/2, get/2, put/2, delete/2, 21 | check_node/2, route/2 22 | ]). 23 | 24 | -include("kai.hrl"). 25 | 26 | start_link() -> 27 | kai_tcp_server:start_link( 28 | {local, ?MODULE}, 29 | ?MODULE, 30 | [], 31 | #tcp_server_option{ 32 | listen = [binary, {packet, 4}, {active, true}, {reuseaddr, true}], 33 | port = kai_config:get(rpc_port), 34 | max_processes = kai_config:get(rpc_max_processes) 35 | } 36 | ). 37 | 38 | stop() -> kai_tcp_server:stop(?MODULE). 39 | 40 | init(_Args) -> {ok, {}}. 41 | 42 | handle_call(Socket, Data, State) -> 43 | dispatch(Socket, binary_to_term(Data), State). 44 | 45 | dispatch(_Socket, node_info, State) -> 46 | reply(kai_config:node_info(), State); 47 | 48 | dispatch(_Socket, node_list, State) -> 49 | reply(kai_hash:node_list(), State); 50 | 51 | dispatch(_Socket, {list, Bucket}, State) -> 52 | reply(kai_store:list(Bucket), State); 53 | 54 | dispatch(_Socket, {get, Data}, State) -> 55 | reply(kai_store:get(Data), State); 56 | 57 | dispatch(_Socket, {put, Data}, State) when is_record(Data, data)-> 58 | reply(kai_store:put(Data), State); 59 | 60 | dispatch(_Socket, {delete, Data}, State) -> 61 | reply(kai_store:delete(Data), State); 62 | 63 | dispatch(_Socket, {check_node, Node}, State) -> 64 | reply(kai_membership:check_node(Node), State); 65 | 66 | dispatch(_Socket, {route, Request}, State) -> 67 | reply(kai_coordinator:route(Request), State); 68 | 69 | dispatch(_Socket, _Unknown, State) -> 70 | reply({error, enotsup}, State). 71 | 72 | reply(Data, State) -> 73 | {reply, term_to_binary(Data), State}. 74 | 75 | recv_response(ApiSocket) -> 76 | receive 77 | {tcp, ApiSocket, Bin} -> 78 | {ok, binary_to_term(Bin)}; 79 | {tcp_closed, ApiSocket} -> 80 | {error, econnreset}; 81 | {error, Reason} -> 82 | {error, Reason} 83 | 84 | % Don't place Other alternative here. This is to avoid to catch event 85 | % messages, '$gen_event' or something like that. Remember that this 86 | % function can be called from gen_fsm/gen_event. 87 | 88 | after ?TIMEOUT -> 89 | {error, timeout} 90 | end. 91 | 92 | do_request(Node, Message) -> 93 | case kai_connection:lease(Node, self()) of 94 | {ok, ApiSocket} -> 95 | case gen_tcp:send(ApiSocket, term_to_binary(Message)) of 96 | ok -> 97 | case recv_response(ApiSocket) of 98 | {ok, Result} -> 99 | kai_connection:return(ApiSocket), 100 | {ok, Result}; 101 | {error, Reason} -> 102 | kai_connection:close(ApiSocket), 103 | {error, Reason} 104 | end; 105 | {error, Reason} -> 106 | kai_connection:close(ApiSocket), 107 | {error, Reason} 108 | end; 109 | {error, Reason} -> 110 | {error, Reason} 111 | end. 112 | 113 | request(Node, Message) -> 114 | case do_request(Node, Message) of 115 | {ok, Result} -> 116 | Result; 117 | {error, Reason} -> 118 | ?warning(io_lib:format("request(~p, ~p): ~p", 119 | [Node, Message, {error, Reason}])), 120 | % kai_membership:check_node(Node), 121 | {error, Reason} 122 | end. 123 | 124 | is_local_node(Node) -> 125 | LocalNode = kai_config:get(node), 126 | Node =:= LocalNode. 127 | 128 | node_info(Node) -> 129 | case is_local_node(Node) of 130 | true -> kai_config:node_info(); 131 | _ -> request(Node, node_info) 132 | end. 133 | 134 | node_list(Node) -> 135 | case is_local_node(Node) of 136 | true -> kai_hash:node_list(); 137 | _ -> request(Node, node_list) 138 | end. 139 | 140 | list(Node, Bucket) -> 141 | case is_local_node(Node) of 142 | true -> kai_store:list(Bucket); 143 | _ -> request(Node, {list, Bucket}) 144 | end. 145 | 146 | get(Node, Data) -> 147 | case is_local_node(Node) of 148 | true -> kai_store:get(Data); 149 | _ -> request(Node, {get, Data}) 150 | end. 151 | 152 | put(Node, Data) -> 153 | case is_local_node(Node) of 154 | true -> kai_store:put(Data); 155 | _ -> request(Node, {put, Data}) 156 | end. 157 | 158 | delete(Node, Data) -> 159 | case is_local_node(Node) of 160 | true -> kai_store:delete(Data); 161 | _ -> request(Node, {delete, Data}) 162 | end. 163 | 164 | check_node(Node, Node2) -> 165 | case is_local_node(Node) of 166 | true -> kai_membership:check_node(Node2); 167 | _ -> request(Node, {check_node, Node2}) 168 | end. 169 | 170 | route(Node, Request) -> 171 | case is_local_node(Node) of 172 | true -> {error, ewouldblock}; 173 | _ -> request(Node, {route, Request}) 174 | end. 175 | -------------------------------------------------------------------------------- /src/kai_stat.erl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -module(kai_stat). 14 | -behaviour(gen_server). 15 | 16 | -export([start_link/0, stop/0]). 17 | -export([all/0, incr_cmd_get/0, incr_cmd_set/0, add_bytes_read/1, 18 | add_bytes_write/1, incr_unreconciled_get/1]). 19 | -export([ 20 | init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, 21 | code_change/3 22 | ]). 23 | 24 | -include("kai.hrl"). 25 | -record(state, { 26 | boot_time, cmd_get, cmd_set, bytes_read, bytes_write, 27 | unreconciled_get, node, quorum, 28 | number_of_buckets, number_of_virtual_nodes, store 29 | }). 30 | 31 | -define(SERVER, ?MODULE). 32 | 33 | start_link() -> 34 | gen_server:start_link({local, ?SERVER}, ?MODULE, [], _Opts = []). 35 | 36 | init(_Args) -> 37 | {Msec, Sec, _Usec} = now(), 38 | BootTime = 1000000 * Msec + Sec, 39 | [LocalNode, N, R, W, NumberOfBuckets, NumberOfVirtualNodes, Store] = 40 | kai_config:get([ 41 | node, n, r, w, number_of_buckets, number_of_virtual_nodes, store 42 | ]), 43 | Quorum = join(",", [integer_to_list(X) || X <- [N, R, W]]), 44 | {ok, #state{ 45 | boot_time = BootTime, 46 | cmd_get = 0, 47 | cmd_set = 0, 48 | bytes_read = 0, 49 | bytes_write = 0, 50 | unreconciled_get = array:new([{size, N-1}, {fixed, false}, 51 | {default, [0,0]}]), 52 | node = LocalNode, 53 | quorum = Quorum, 54 | number_of_buckets = NumberOfBuckets, 55 | number_of_virtual_nodes = NumberOfVirtualNodes, 56 | store = Store 57 | }}. 58 | 59 | terminate(_Reason, _State) -> 60 | ok. 61 | 62 | join(_Delimiter, [], Acc) -> 63 | Acc; 64 | join(_Delimiter, [Token], Acc) -> 65 | Acc ++ Token; 66 | join(Delimiter, [Token|Rest], Acc) -> 67 | join(Delimiter, Rest, Acc ++ Token ++ Delimiter). 68 | join(Delimiter, List) -> 69 | join(Delimiter, List, []). 70 | 71 | node_to_list({{A1,A2,A3,A4}, Port}) -> 72 | Addr = join(".", [integer_to_list(X) || X <- [A1,A2,A3,A4]]), 73 | Addr ++ ":" ++ integer_to_list(Port). 74 | 75 | format_unreconciled_get(UnreconciledGet) -> 76 | join(" ", 77 | array:foldr( 78 | fun(_Index, [UnReconciled, InternalNum], Acc) -> 79 | [io_lib:format("~B(~B)", [UnReconciled, InternalNum]) | Acc] 80 | end, [], UnreconciledGet)). 81 | 82 | stat(Name, State) -> 83 | case Name of 84 | uptime -> 85 | {Msec, Sec, _Usec} = now(), 86 | Uptime = 1000000 * Msec + Sec - State#state.boot_time, 87 | {uptime, integer_to_list(Uptime)}; 88 | time -> 89 | {Msec, Sec, _Usec} = now(), 90 | Time = 1000000 * Msec + Sec, 91 | {time, integer_to_list(Time)}; 92 | version -> 93 | Version = 94 | case application:get_key(kai, vsn) of 95 | {ok, V} -> V; 96 | _ -> "0" 97 | end, 98 | {version, Version}; 99 | bytes -> 100 | Bytes = kai_store:info(bytes), 101 | {bytes, integer_to_list(Bytes)}; 102 | curr_items -> 103 | Size = kai_store:info(size), 104 | {curr_items, integer_to_list(Size)}; 105 | curr_connections -> 106 | Connections = kai_tcp_server:info(kai_memcache, curr_connections), 107 | {curr_connections, integer_to_list(Connections)}; 108 | cmd_get -> 109 | {cmd_get, integer_to_list(State#state.cmd_get)}; 110 | cmd_set -> 111 | {cmd_set, integer_to_list(State#state.cmd_set)}; 112 | bytes_read -> 113 | {bytes_read, integer_to_list(State#state.bytes_read)}; 114 | bytes_write -> 115 | {bytes_write, integer_to_list(State#state.bytes_write)}; 116 | kai_node -> 117 | {kai_node, node_to_list(State#state.node)}; 118 | kai_quorum -> 119 | {kai_quorum, State#state.quorum}; 120 | kai_number_of_buckets -> 121 | NumberOfBuckets = State#state.number_of_buckets, 122 | {kai_number_of_buckets, integer_to_list(NumberOfBuckets)}; 123 | kai_number_of_virtual_nodes -> 124 | NumberOfVirtualNodes = State#state.number_of_virtual_nodes, 125 | {kai_number_of_virtual_nodes, 126 | integer_to_list(NumberOfVirtualNodes)}; 127 | kai_store -> 128 | {kai_store, atom_to_list(State#state.store)}; 129 | kai_curr_nodes -> 130 | {node_list, Nodes} = kai_hash:node_list(), 131 | NodesInList = lists:sort([node_to_list(N) || N <- Nodes]), 132 | {kai_curr_nodes, join(" ", NodesInList)}; 133 | kai_unreconciled_get -> 134 | {kai_unreconciled_get, 135 | format_unreconciled_get(State#state.unreconciled_get)}; 136 | % kai_curr_buckets -> 137 | % {buckets, Buckets} = kai_hash:buckets(), 138 | % {kai_curr_buckets, join(" ", [integer_to_list(B) || B <- Buckets])}; 139 | erlang_procs -> 140 | ErlangProcs = erlang:system_info(process_count), 141 | {erlang_procs, integer_to_list(ErlangProcs)}; 142 | erlang_version -> 143 | {erlang_version, erlang:system_info(version)} 144 | end. 145 | 146 | all(State) -> 147 | Stats = 148 | lists:map( 149 | fun(Name) -> stat(Name, State) end, 150 | [uptime, time, version, bytes, 151 | curr_items, curr_connections, cmd_get, cmd_set, 152 | bytes_read, bytes_write, 153 | kai_node, kai_quorum, kai_number_of_buckets, 154 | kai_number_of_virtual_nodes, kai_store, kai_curr_nodes, 155 | kai_unreconciled_get, 156 | %kai_curr_buckets, 157 | erlang_procs, erlang_version] 158 | ), 159 | {reply, Stats, State}. 160 | 161 | incr_cmd_get(State) -> 162 | State2 = State#state{cmd_get = State#state.cmd_get + 1}, 163 | {noreply, State2}. 164 | 165 | incr_cmd_set(State) -> 166 | State2 = State#state{cmd_set = State#state.cmd_set + 1}, 167 | {noreply, State2}. 168 | 169 | incr_unreconciled_get(InternalNum, Reconciled, State) -> 170 | Old = State#state.unreconciled_get, 171 | Index = InternalNum -2, % index is 0: two versions, 1: three, etc. 172 | [Unreconciled, Internal] = array:get(Index, Old), 173 | New = array:set(InternalNum -2, 174 | case Reconciled of 175 | true -> [Unreconciled, Internal + 1]; 176 | _ -> [Unreconciled + 1, Internal + 1] 177 | end, 178 | Old), 179 | State2 = State#state{unreconciled_get = New}, 180 | {noreply, State2}. 181 | 182 | add_bytes_read(Data, State) -> 183 | Bytes = lists:sum([byte_size(D#data.value) || D <- Data]), 184 | State2 = State#state{bytes_read = State#state.bytes_read + Bytes}, 185 | {noreply, State2}. 186 | 187 | add_bytes_write(Data, State) -> 188 | Bytes = byte_size(Data#data.value), 189 | State2 = State#state{bytes_write = State#state.bytes_write + Bytes}, 190 | {noreply, State2}. 191 | 192 | handle_call(stop, _From, State) -> 193 | {stop, normal, stopped, State}; 194 | handle_call(all, _From, State) -> 195 | all(State). 196 | handle_cast(incr_cmd_get, State) -> 197 | incr_cmd_get(State); 198 | handle_cast(incr_cmd_set, State) -> 199 | incr_cmd_set(State); 200 | handle_cast({add_bytes_read, Data}, State) -> 201 | add_bytes_read(Data, State); 202 | handle_cast({add_bytes_write, Data}, State) -> 203 | add_bytes_write(Data, State); 204 | handle_cast({incr_unreconciled_get, {InternalNum, Reconciled}}, State) -> 205 | incr_unreconciled_get(InternalNum, Reconciled, State). 206 | handle_info(_Info, State) -> 207 | {noreply, State}. 208 | code_change(_OldVsn, State, _Extra) -> 209 | {ok, State}. 210 | 211 | stop() -> 212 | gen_server:call(?SERVER, stop). 213 | all() -> 214 | gen_server:call(?SERVER, all). 215 | incr_cmd_get() -> 216 | gen_server:cast(?SERVER, incr_cmd_get). 217 | incr_cmd_set() -> 218 | gen_server:cast(?SERVER, incr_cmd_set). 219 | add_bytes_read(Data) -> 220 | gen_server:cast(?SERVER, {add_bytes_read, Data}). 221 | add_bytes_write(Data) -> 222 | gen_server:cast(?SERVER, {add_bytes_write, Data}). 223 | incr_unreconciled_get(Data) -> 224 | gen_server:cast(?SERVER, {incr_unreconciled_get, Data}). 225 | -------------------------------------------------------------------------------- /src/kai_store.erl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -module(kai_store). 14 | 15 | -export([start_link/0, stop/0]). 16 | -export([list/1, get/1, put/1, delete/1, info/1, match/1]). 17 | -compile(export_all). 18 | -include("kai.hrl"). 19 | 20 | -define(SERVER, ?MODULE). 21 | 22 | start_link() -> 23 | Store = kai_config:get(store), 24 | Module = list_to_atom(atom_to_list(?MODULE) ++ "_" ++ atom_to_list(Store)), 25 | apply(Module, start_link, [?SERVER]). 26 | 27 | stop() -> 28 | gen_server:call(?SERVER, stop). 29 | list(Bucket) -> 30 | gen_server:call(?SERVER, {list, Bucket}). 31 | get(Data) -> 32 | gen_server:call(?SERVER, {get, Data}). 33 | match(Data) -> 34 | gen_server:call(?SERVER, {match, Data}). 35 | put(Data) -> 36 | gen_server:call(?SERVER, {put, Data}). 37 | delete(Data) -> 38 | gen_server:call(?SERVER, {delete, Data}). 39 | info(Name) -> 40 | gen_server:call(?SERVER, {info, Name}). 41 | bucket(Name) -> 42 | gen_server:call(?SERVER, {bucket, Name}). 43 | -------------------------------------------------------------------------------- /src/kai_store_dets.erl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -module(kai_store_dets). 14 | -compile(export_all). 15 | -behaviour(gen_server). 16 | 17 | -export([start_link/1]). 18 | -export([ 19 | init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, 20 | code_change/3 21 | ]). 22 | 23 | -include("kai.hrl"). 24 | -record(state, {number_of_tables, tables}). 25 | 26 | start_link(Server) -> 27 | gen_server:start_link({local, Server}, ?MODULE, [], _Opts = []). 28 | 29 | init(_Args) -> 30 | Dir = kai_config:get(dets_dir), 31 | NumberOfTables = kai_config:get(number_of_tables), 32 | Tables = 33 | lists:map( 34 | fun(I) -> 35 | Name = list_to_atom(atom_to_list(?MODULE) ++ "_" ++ integer_to_list(I)), 36 | File = Dir ++ "/" ++ integer_to_list(I), 37 | case dets:open_file(Name, [{type, set}, {keypos, 2}, {file, File}]) of 38 | {ok, Table} -> {I, Table}; 39 | {error, Reason} -> ?info(Reason), 40 | exit(Reason) 41 | end 42 | end, 43 | lists:seq(1, NumberOfTables) 44 | ), 45 | {ok, #state{number_of_tables = NumberOfTables, tables = Tables}}. 46 | 47 | terminate(_Reason, State) -> 48 | lists:foreach( 49 | fun({_I, Table}) -> dets:close(Table) end, 50 | State#state.tables 51 | ), 52 | ok. 53 | 54 | bucket_to_table(Bucket, State) -> 55 | I = Bucket rem State#state.number_of_tables + 1, 56 | proplists:get_value(I, State#state.tables). 57 | 58 | do_list(Bucket, State) -> 59 | Table = bucket_to_table(Bucket, State), 60 | Head = #data{ 61 | key = '$1', 62 | bucket = Bucket, 63 | last_modified = '$2', 64 | vector_clocks = '$3', 65 | checksum = '$4', 66 | flags = '_', 67 | value = '_' 68 | }, 69 | Cond = [], 70 | Body = [{#data{ 71 | key = '$1', 72 | bucket = Bucket, 73 | last_modified = '$2', 74 | vector_clocks = '$3', 75 | checksum = '$4' 76 | }}], 77 | ListOfData = dets:select(Table, [{Head, Cond, Body}]), 78 | {reply, {list_of_data, ListOfData}, State}. 79 | 80 | do_get(#data{key=Key, bucket=Bucket} = _Data, State) -> 81 | Table = bucket_to_table(Bucket, State), 82 | case dets:lookup(Table, Key) of 83 | [Data] -> {reply, Data, State}; 84 | _ -> {reply, undefined, State} 85 | end. 86 | 87 | do_match(#data{key=Key, bucket=Bucket} = _Data, State) -> 88 | Table = bucket_to_table(Bucket, State), 89 | Data = dets:match(Table, #data{key=Key, bucket=Bucket, value='$1'}), 90 | {reply, Data, State}. 91 | 92 | do_put(Data, State) when is_record(Data, data) -> 93 | Table = bucket_to_table(Data#data.bucket, State), 94 | case dets:lookup(Table, Data#data.key) of 95 | [StoredData] -> 96 | case vclock:descends(Data#data.vector_clocks, StoredData#data.vector_clocks) of 97 | true -> insert_and_reply(Data, Table, State); 98 | _ -> {reply, {error, "stale or concurrent state found in kai_store"}, State} 99 | end; 100 | _ -> insert_and_reply(Data, Table, State) 101 | end. 102 | 103 | insert_and_reply(Data, Table, State) -> 104 | dets:insert(Table, Data), 105 | dets:sync(Table), 106 | {reply, ok, State}. 107 | 108 | do_delete(#data{key=Key, bucket=Bucket} = _Data, State) -> 109 | Table = bucket_to_table(Bucket, State), 110 | case dets:lookup(Table, Key) of 111 | [_Data2] -> 112 | dets:delete(Table, Key), 113 | dets:sync(Table), 114 | {reply, ok, State}; 115 | _ -> 116 | {reply, undefined, State} 117 | end. 118 | 119 | info(Name, State) -> 120 | Values = 121 | lists:map( 122 | fun(I) -> 123 | T = proplists:get_value(I, State#state.tables), 124 | case Name of 125 | bytes -> dets:info(T, file_size); 126 | size -> dets:info(T, size) 127 | end 128 | end, 129 | lists:seq(1, State#state.number_of_tables) 130 | ), 131 | io:format("DETS: ~p~n",[Values]), 132 | {reply, lists:sum(Values), State}. 133 | 134 | handle_call(stop, _From, State) -> 135 | {stop, normal, stopped, State}; 136 | handle_call({list, Bucket}, _From, State) -> 137 | do_list(Bucket, State); 138 | handle_call({get, Data}, _From, State) -> 139 | do_get(Data, State); 140 | handle_call({match, Data}, _From, State) -> 141 | do_match(Data, State); 142 | handle_call({put, Data}, _From, State) -> 143 | do_put(Data, State); 144 | handle_call({delete, Data}, _From, State) -> 145 | do_delete(Data, State); 146 | handle_call({bucket, Name}, _From, State) -> 147 | {reply,bucket_to_table(Name, State),State}; 148 | handle_call({info, Name}, _From, State) -> 149 | info(Name, State). 150 | handle_cast(_Msg, State) -> 151 | {noreply, State}. 152 | handle_info(_Info, State) -> 153 | {noreply, State}. 154 | code_change(_OldVsn, State, _Extra) -> 155 | {ok, State}. 156 | -------------------------------------------------------------------------------- /src/kai_store_ets.erl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -module(kai_store_ets). 14 | -behaviour(gen_server). 15 | 16 | -export([start_link/1]). 17 | -export([ 18 | init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, 19 | code_change/3 20 | ]). 21 | 22 | -include("kai.hrl"). 23 | 24 | start_link(Server) -> 25 | gen_server:start_link({local, Server}, ?MODULE, [], _Opts = []). 26 | 27 | init(_Args) -> 28 | ets:new(?MODULE, [set, private, named_table, {keypos, 2}]), 29 | {ok, []}. 30 | 31 | terminate(_Reason, _State) -> 32 | ets:delete(?MODULE), 33 | ok. 34 | 35 | do_list(Bucket, State) -> 36 | Head = #data{ 37 | key = '$1', 38 | bucket = Bucket, 39 | last_modified = '$2', 40 | vector_clocks = '$3', 41 | checksum = '$4', 42 | flags = '_', 43 | value = '_' 44 | }, 45 | Cond = [], 46 | Body = [{#data{ 47 | key = '$1', 48 | bucket = Bucket, 49 | last_modified = '$2', 50 | vector_clocks = '$3', 51 | checksum = '$4' 52 | }}], 53 | ListOfData = ets:select(?MODULE, [{Head, Cond, Body}]), 54 | {reply, {list_of_data, ListOfData}, State}. 55 | 56 | do_get(#data{key=Key} = _Data, State) -> 57 | case ets:lookup(?MODULE, Key) of 58 | [Data] -> {reply, Data, State}; 59 | _ -> {reply, undefined, State} 60 | end. 61 | 62 | do_match(#data{key=Key} = _Data, State) -> 63 | case ets:match(?MODULE, Key) of 64 | [Data] -> {reply, Data, State}; 65 | _ -> {reply, undefined, State} 66 | end. 67 | 68 | do_put(Data, State) when is_record(Data, data) -> 69 | case ets:lookup(?MODULE, Data#data.key) of 70 | [StoredData] -> 71 | case vclock:descends(Data#data.vector_clocks, StoredData#data.vector_clocks) of 72 | true -> insert_and_reply(Data, State); 73 | _ -> {reply, {error, "stale or concurrent state found in kai_store"}, State} 74 | end; 75 | _ -> insert_and_reply(Data, State) 76 | end; 77 | do_put(a, b) -> ok. 78 | 79 | insert_and_reply(Data, State) -> 80 | ets:insert(?MODULE, Data), 81 | {reply, ok, State}. 82 | 83 | do_delete(#data{key=Key} = _Data, State) -> 84 | case ets:lookup(?MODULE, Key) of 85 | [_Data2] -> 86 | ets:delete(?MODULE, Key), 87 | {reply, ok, State}; 88 | _ -> 89 | {reply, undefined, State} 90 | end. 91 | 92 | info(Name, State) -> 93 | Value = 94 | case Name of 95 | bytes -> 96 | % this code roughly estimates the size of stored objects, 97 | % since ets only store a reference to the binary 98 | Ets = erlang:system_info(wordsize) + ets:info(?MODULE, memory), 99 | Bin = erlang:memory(binary), 100 | Ets + Bin; 101 | size -> 102 | ets:info(?MODULE, size) 103 | end, 104 | {reply, Value, State}. 105 | 106 | handle_call(stop, _From, State) -> 107 | {stop, normal, stopped, State}; 108 | handle_call({list, Bucket}, _From, State) -> 109 | do_list(Bucket, State); 110 | handle_call({get, Data}, _From, State) -> 111 | do_get(Data, State); 112 | handle_call({match, Data}, _From, State) -> 113 | do_match(Data, State); 114 | handle_call({put, Data}, _From, State) -> 115 | do_put(Data, State); 116 | handle_call({delete, Data}, _From, State) -> 117 | do_delete(Data, State); 118 | handle_call({info, Name}, _From, State) -> 119 | info(Name, State). 120 | handle_cast(_Msg, State) -> 121 | {noreply, State}. 122 | handle_info(_Info, State) -> 123 | {noreply, State}. 124 | code_change(_OldVsn, State, _Extra) -> 125 | {ok, State}. 126 | -------------------------------------------------------------------------------- /src/kai_sup.erl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -module(kai_sup). 14 | -behaviour(supervisor). 15 | 16 | -export([start_link/1]). 17 | -export([init/1]). 18 | 19 | -define(SERVER, ?MODULE). 20 | 21 | start_link(Args) -> 22 | supervisor:start_link({local, ?SERVER}, ?MODULE, Args). 23 | 24 | init(Args) -> 25 | Config = { 26 | kai_config, 27 | {kai_config, start_link, [Args]}, 28 | permanent, 1000, worker, 29 | [kai_config] 30 | }, 31 | Log = { 32 | kai_log, 33 | {kai_log, start_link, []}, 34 | permanent, 1000, worker, 35 | [kai_log] 36 | }, 37 | Hash = { 38 | kai_hash, 39 | {kai_hash, start_link, []}, 40 | permanent, 1000, worker, 41 | [kai_hash] 42 | }, 43 | Store = { 44 | kai_store, 45 | {kai_store, start_link, []}, 46 | permanent, 1000, worker, 47 | [kai_store] 48 | }, 49 | Stat = { 50 | kai_stat, 51 | {kai_stat, start_link, []}, 52 | permanent, 1000, worker, 53 | [kai_stat] 54 | }, 55 | Version = { 56 | kai_version, 57 | {kai_version, start_link, []}, 58 | permanent, 1000, worker, 59 | [kai_version] 60 | }, 61 | Connection = { 62 | kai_connection, 63 | {kai_connection, start_link, []}, 64 | permanent, 1000, worker, 65 | [kai_connection] 66 | }, 67 | Sync = { 68 | kai_sync, 69 | {kai_sync, start_link, []}, 70 | permanent, 1000, worker, 71 | [kai_sync] 72 | }, 73 | Membership = { 74 | kai_membership, 75 | {kai_membership, start_link, []}, 76 | permanent, 1000, worker, 77 | [kai_membership] 78 | }, 79 | Rpc = { 80 | kai_rpc, 81 | {kai_rpc, start_link, []}, 82 | permanent, 1000, worker, 83 | [kai_rpc] 84 | }, 85 | Memcache = { 86 | kai_memcache, 87 | {kai_memcache, start_link, []}, 88 | permanent, 1000, worker, 89 | [kai_memcache] 90 | }, 91 | {ok, {{one_for_one, 3, 10}, [ 92 | Config, Log, Hash, Store, Stat, Version, Connection, Sync, Membership, 93 | Rpc, Memcache 94 | ]}}. 95 | -------------------------------------------------------------------------------- /src/kai_sync.erl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -module(kai_sync). 14 | -behaviour(gen_fsm). 15 | 16 | -export([start_link/0, stop/0]). 17 | -export([update_bucket/1, delete_bucket/1]). 18 | -export([ 19 | init/1, ready/2, handle_event/3, handle_sync_event/4, handle_info/3, 20 | terminate/3, code_change/4 21 | ]). 22 | 23 | -include("kai.hrl"). 24 | 25 | -define(SERVER, ?MODULE). 26 | 27 | start_link() -> 28 | gen_fsm:start_link({local, ?SERVER}, ?MODULE, [], _Opts = []). 29 | 30 | init(_Args) -> 31 | {ok, ready, [], ?TIMER}. 32 | 33 | terminate(_Reason, _StateName, _StateData) -> 34 | ok. 35 | 36 | retrieve_data(_Node, []) -> 37 | ok; 38 | retrieve_data(Node, [Metadata|Rest]) -> 39 | case kai_store:get(Metadata) of 40 | Data when is_record(Data, data) -> 41 | retrieve_data(Node, Rest); 42 | undefined -> 43 | case kai_rpc:get(Node, Metadata) of 44 | Data when is_record(Data, data) -> 45 | kai_store:put(Data), 46 | retrieve_data(Node, Rest); 47 | undefined -> 48 | retrieve_data(Node, Rest); 49 | {error, Reason} -> 50 | ?warning(io_lib:format("retrieve_data/2: ~p", [{error, Reason}])), 51 | {error, Reason} 52 | end 53 | end. 54 | 55 | do_update_bucket(_Bucket, []) -> 56 | {error, enodata}; 57 | do_update_bucket(Bucket, [Node|Rest]) -> 58 | case kai_rpc:list(Node, Bucket) of 59 | {list_of_data, ListOfData} -> 60 | retrieve_data(Node, ListOfData); 61 | {error, Reason} -> 62 | ?warning(io_lib:format("do_update_bucket/2: ~p", [{error, Reason}])), 63 | do_update_bucket(Bucket, Rest) 64 | end. 65 | 66 | do_update_bucket(Bucket) -> 67 | {nodes, Nodes} = kai_hash:find_nodes(Bucket), 68 | LocalNode = kai_config:get(node), 69 | do_update_bucket(Bucket, Nodes -- [LocalNode]). 70 | 71 | do_delete_bucket([]) -> 72 | ok; 73 | do_delete_bucket([Metadata|Rest]) -> 74 | kai_store:delete(Metadata), 75 | do_delete_bucket(Rest); 76 | do_delete_bucket(Bucket) -> 77 | {list_of_data, ListOfData} = kai_store:list(Bucket), 78 | do_delete_bucket(ListOfData). 79 | 80 | ready({update_bucket, Bucket}, State) -> 81 | do_update_bucket(Bucket), 82 | {next_state, ready, State, ?TIMER}; 83 | ready({delete_bucket, Bucket}, State) -> 84 | do_delete_bucket(Bucket), 85 | {next_state, ready, State, ?TIMER}; 86 | ready(timeout, State) -> 87 | case kai_hash:choose_bucket_randomly() of 88 | {bucket, Bucket} -> do_update_bucket(Bucket); 89 | _ -> nop 90 | end, 91 | {next_state, ready, State, ?TIMER}. 92 | 93 | handle_event(stop, _StateName, StateData) -> 94 | {stop, normal, StateData}. 95 | handle_sync_event(_Event, _From, _StateName, StateData) -> 96 | {next_state, wait, StateData, ?TIMEOUT}. 97 | handle_info(_Info, _StateName, StateData) -> 98 | {next_state, ready, StateData, ?TIMER}. 99 | code_change(_OldVsn, _StateName, StateData, _Extra) -> 100 | {ok, ready, StateData}. 101 | 102 | stop() -> 103 | gen_fsm:send_all_state_event(?SERVER, stop). 104 | update_bucket(Bucket) -> 105 | gen_fsm:send_event(?SERVER, {update_bucket, Bucket}). 106 | delete_bucket(Bucket) -> 107 | gen_fsm:send_event(?SERVER, {delete_bucket, Bucket}). 108 | -------------------------------------------------------------------------------- /src/kai_tcp_server.erl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -module(kai_tcp_server). 14 | 15 | -export([behaviour_info/1]). 16 | -export([start_link/1, start_link/2, start_link/3, start_link/4]). 17 | -export([stop/0, stop/1]). 18 | -export([info/1, info/2]). 19 | 20 | -include("kai.hrl"). 21 | 22 | % Behaviour Callbacks 23 | behaviour_info(callbacks) -> [{init, 1}, {handle_call, 3}]; 24 | behaviour_info(_Other) -> undefined. 25 | 26 | % External APIs 27 | start_link(Mod) -> start_link(Mod, []). 28 | start_link(Mod, Args) -> start_link(Mod, Args, #tcp_server_option{}). 29 | start_link(Mod, Args, Option) -> 30 | start_link({local, ?MODULE}, Mod, Args, Option). 31 | start_link(Name, Mod, Args, Option) -> 32 | kai_tcp_server_sup:start_link(Name, Mod, Args, Option). 33 | 34 | stop() -> stop(?MODULE). 35 | stop(Name) -> 36 | kai_tcp_server_sup:stop(Name). 37 | 38 | info(Key) -> info(?MODULE, Key). 39 | info(Name, Key) -> 40 | kai_tcp_server_monitor:info( 41 | kai_tcp_server_sup:build_monitor_name(Name), Key 42 | ). 43 | 44 | -------------------------------------------------------------------------------- /src/kai_tcp_server_acceptor.erl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -module(kai_tcp_server_acceptor). 14 | 15 | -export([start_link/6]). 16 | -export([init/6]). 17 | 18 | -include("kai.hrl"). 19 | 20 | % External APIs 21 | start_link({Dest, Name}, ListenSocket, State, MonitorName, Mod, Option) -> 22 | {ok, Pid} = proc_lib:start_link( 23 | ?MODULE, init, 24 | [self(), ListenSocket, State, MonitorName, Mod, Option] 25 | ), 26 | case Dest of 27 | local -> register(Name, Pid); 28 | _Global -> global:register_name(Name, Pid) 29 | end, 30 | {ok, Pid}. 31 | 32 | % Callbacks 33 | init(Parent, ListenSocket, State, MonitorName, Mod, Option) -> 34 | proc_lib:init_ack(Parent, {ok, self()}), 35 | kai_tcp_server_monitor:register(MonitorName, self()), 36 | accept(ListenSocket, State, MonitorName, Mod, Option). 37 | 38 | % Internal Functions 39 | accept(ListenSocket, State, MonitorName, Mod, Option) -> 40 | case gen_tcp:accept( 41 | ListenSocket, Option#tcp_server_option.accept_timeout 42 | ) of 43 | {ok, Socket} -> 44 | try 45 | kai_tcp_server_monitor:increment(MonitorName, self()), 46 | recv( 47 | proplists:get_value( 48 | active, Option#tcp_server_option.listen 49 | ), 50 | Socket, State, Mod, Option 51 | ) 52 | catch 53 | Type:Reason -> 54 | ?warning(io_lib:format( 55 | "accept(~p) ~p", [Mod, {Type, Reason}] 56 | )) 57 | after 58 | kai_tcp_server_monitor:decrement(MonitorName, self()), 59 | gen_tcp:close(Socket) 60 | end; 61 | {error, Reason} -> 62 | ?warning(io_lib:format( 63 | "accept(~p) ~p", [Mod, {error, Reason}] 64 | )), 65 | timer:sleep(Option#tcp_server_option.accept_error_sleep_time) 66 | end, 67 | accept(ListenSocket, State, MonitorName, Mod, Option). 68 | 69 | recv(false, Socket, State, Mod, Option) -> 70 | case gen_tcp:recv( 71 | Socket, 72 | Option#tcp_server_option.recv_length, 73 | Option#tcp_server_option.recv_timeout 74 | ) of 75 | {ok, Data} -> 76 | call_mod(false, Socket, Data, State, Mod, Option); 77 | {error, closed} -> 78 | tcp_closed; 79 | {error, Reason} -> 80 | ?warning(io_lib:format("recv(~p) ~p", [Mod, {error, Reason}])), 81 | error 82 | end; 83 | 84 | recv(true, _DummySocket, State, Mod, Option) -> 85 | receive 86 | {tcp, Socket, Data} -> 87 | call_mod(true, Socket, Data, State, Mod, Option); 88 | {tcp_closed, _Socket} -> 89 | tcp_closed; 90 | Error -> 91 | ?warning(io_lib:format("recv(~p) ~p", [Mod, {error, Error}])), 92 | error 93 | after Option#tcp_server_option.recv_timeout -> 94 | tcp_timeout 95 | end. 96 | 97 | call_mod(Active, Socket, Data, State, Mod, Option) -> 98 | case Mod:handle_call(Socket, Data, State) of 99 | {reply, DataToSend, State} -> 100 | gen_tcp:send(Socket, DataToSend), 101 | recv(Active, Socket, State, Mod, Option); 102 | {noreply, State} -> 103 | recv(Active, Socket, State, Mod, Option); 104 | {close, State} -> 105 | tcp_closed; 106 | {close, DataToSend, State} -> 107 | gen_tcp:send(Socket, DataToSend); 108 | Other -> 109 | ?warning(io_lib:format( 110 | "call_mod(~p) ~p", [Mod, {unexpected_result, Other}] 111 | )) 112 | end. 113 | 114 | -------------------------------------------------------------------------------- /src/kai_tcp_server_monitor.erl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -module(kai_tcp_server_monitor). 14 | -behaviour(gen_server). 15 | 16 | -export([start_link/1, stop/1]). 17 | -export([ 18 | register/2, 19 | increment/2, decrement/2, 20 | info/2 21 | ]). 22 | -export([ 23 | init/1, 24 | handle_call/3, handle_cast/2, handle_info/2, 25 | terminate/2, code_change/3 26 | ]). 27 | 28 | -include("kai.hrl"). 29 | 30 | % External APIs 31 | start_link(Name) -> 32 | gen_server:start_link(Name, ?MODULE, [], []). 33 | 34 | stop(ServerRef) -> 35 | gen_server:call(ServerRef, stop). 36 | 37 | register(ServerRef, Pid) -> 38 | gen_server:call(ServerRef, {register, Pid}). 39 | 40 | increment(ServerRef, Pid) -> 41 | gen_server:cast(ServerRef, {increment, Pid}). 42 | 43 | decrement(ServerRef, Pid) -> 44 | gen_server:cast(ServerRef, {decrement, Pid}). 45 | 46 | info(ServerRef, Key) -> 47 | gen_server:call(ServerRef, {info, Key}). 48 | 49 | % Callbacks 50 | init(_Args) -> 51 | {ok, {_MonitorRefs = [], _Pids = []}}. 52 | 53 | handle_call(stop, _From, State) -> 54 | {stop, normal, stopped, State}; 55 | 56 | handle_call({register, Pid}, _From, {MonitorRefs, Pids}) -> 57 | {reply, ok, { 58 | [erlang:monitor(process, Pid) | MonitorRefs], 59 | Pids 60 | }}; 61 | 62 | handle_call({info, Key}, _From, State) -> 63 | {reply, state_to_info(State, Key), State}; 64 | 65 | handle_call(_Message, _From, State) -> 66 | {reply, ok, State}. 67 | 68 | handle_cast({increment, Pid}, {MonitorRefs, Pids}) -> 69 | {noreply, {MonitorRefs, [Pid | Pids]}}; 70 | 71 | handle_cast({decrement, Pid}, {MonitorRefs, Pids}) -> 72 | {noreply, {MonitorRefs, lists:delete(Pid, Pids)}}; 73 | 74 | handle_cast(_Message, State) -> 75 | {noreply, State}. 76 | 77 | handle_info({'DOWN', MonitorRef, _Type, Pid, _Info}, {MonitorRefs, Pids}) -> 78 | erlang:demonitor(MonitorRef), 79 | {noreply, { 80 | lists:delete(MonitorRef, MonitorRefs), 81 | lists:delete(Pid, Pids) 82 | }}; 83 | 84 | handle_info(_Info, State) -> 85 | {noreply, State}. 86 | 87 | terminate(_Reason, _State) -> 88 | ok. 89 | 90 | code_change(_OldVsn, State, _Extra) -> 91 | {ok, State}. 92 | 93 | % Internal Functions 94 | state_to_info({_MonitorRefs, Pids}, curr_connections) -> 95 | length(Pids); 96 | 97 | state_to_info(_State, _Key) -> 98 | undefined. 99 | 100 | -------------------------------------------------------------------------------- /src/kai_tcp_server_sup.erl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -module(kai_tcp_server_sup). 14 | -behaviour(supervisor). 15 | 16 | -export([start_link/4, stop/1]). 17 | -export([init/1]). 18 | -export([build_monitor_name/1]). 19 | 20 | -include("kai.hrl"). 21 | 22 | % External APIs 23 | start_link(Name, Mod, Args, Option) -> 24 | supervisor:start_link(Name, ?MODULE, [Name, Mod, Args, Option]). 25 | 26 | stop(Name) -> 27 | case whereis(Name) of 28 | Pid when is_pid(Pid) -> 29 | exit(Pid, normal), 30 | ok; 31 | _ -> not_started 32 | end. 33 | 34 | % Callbacks 35 | init([Name, Mod, Args, Option]) -> 36 | case Mod:init(Args) of 37 | {ok, State} -> listen(State, Name, Mod, Option); 38 | {stop, Reason} -> Reason; 39 | Other -> Other % 'ignore' is contained. 40 | end. 41 | 42 | % Internal Functions 43 | listen(State, Name, Mod, Option) -> 44 | case gen_tcp:listen( 45 | Option#tcp_server_option.port, 46 | Option#tcp_server_option.listen 47 | ) of 48 | {ok, ListenSocket} -> 49 | build_result(ListenSocket, State, Name, Mod, Option); 50 | {error, Reason} -> 51 | ?warning(io_lib:format("listen(~p) ~p", [Mod, {error, Reason}])), 52 | {stop, Reason} 53 | end. 54 | 55 | build_result(ListenSocket, State, {Dest, Name}, Mod, Option) -> 56 | #tcp_server_option{ 57 | max_restarts = MaxRestarts, 58 | time = Time 59 | } = Option, 60 | MonitorName = build_monitor_name(Name), 61 | {ok, { 62 | {one_for_one, MaxRestarts, Time}, 63 | [ 64 | monitor_spec({Dest, MonitorName}) | 65 | acceptor_specs( 66 | ListenSocket, State, {Dest, Name}, MonitorName, Mod, Option 67 | ) 68 | ] 69 | }}. 70 | 71 | monitor_spec({Dest, MonitorName}) -> 72 | { 73 | MonitorName, 74 | { 75 | kai_tcp_server_monitor, 76 | start_link, 77 | [{Dest, MonitorName}] 78 | }, 79 | permanent, 80 | brutal_kill, 81 | worker, 82 | [] 83 | }. 84 | 85 | acceptor_specs( 86 | ListenSocket, State, {Dest, Name}, MonitorBaseName, Mod, Option 87 | ) -> 88 | #tcp_server_option{ 89 | max_processes = MaxProcesses, 90 | shutdown = Shutdown 91 | } = Option, 92 | MonitorName = case Dest of 93 | local -> MonitorBaseName; 94 | _Global -> {Dest, MonitorBaseName} 95 | end, 96 | lists:map( 97 | fun (N) -> 98 | AcceptorName = build_acceptor_name(Name, N), 99 | { 100 | AcceptorName, 101 | { 102 | kai_tcp_server_acceptor, 103 | start_link, 104 | [ 105 | {Dest, AcceptorName}, 106 | ListenSocket, 107 | State, 108 | MonitorName, 109 | Mod, 110 | Option 111 | ] 112 | }, 113 | permanent, 114 | Shutdown, 115 | worker, 116 | [] 117 | } 118 | end, 119 | lists:seq(1, MaxProcesses) 120 | ). 121 | 122 | build_monitor_name(Prefix) -> 123 | list_to_atom(atom_to_list(Prefix) ++ "_monitor"). 124 | 125 | build_acceptor_name(Prefix, Number) -> 126 | list_to_atom( 127 | atom_to_list(Prefix) ++ "_acceptor_" ++ integer_to_list(Number) 128 | ). 129 | 130 | -------------------------------------------------------------------------------- /src/kai_version.erl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -module(kai_version). 14 | -behaviour(gen_server). 15 | 16 | -export([start_link/0, stop/0]). 17 | -export([update/1, order/1, cas_unique/1]). 18 | -export([ 19 | init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, 20 | code_change/3 21 | ]). 22 | 23 | -include("kai.hrl"). 24 | -record(state, {vector_clocks}). 25 | 26 | -define(SERVER, ?MODULE). 27 | -define(CAS_UNIQUE_BITS, 64). 28 | start_link() -> 29 | gen_server:start_link({local, ?SERVER}, ?MODULE, [], _Opts = []). 30 | 31 | init(_Args) -> 32 | NodeVClock = vclock:increment(kai_config:get(node), vclock:fresh()), 33 | {ok, #state{vector_clocks = NodeVClock}}. 34 | 35 | terminate(_Reason, _State) -> 36 | ok. 37 | 38 | update(Data, State) -> 39 | NodeVClock =State#state.vector_clocks, 40 | NewNodeVClock = vclock:increment(kai_config:get(node), NodeVClock), 41 | NewDataVClock = vclock:increment(kai_config:get(node), Data#data.vector_clocks), 42 | NewState = State#state{vector_clocks = NewNodeVClock}, 43 | {reply, 44 | {ok, Data#data{last_modified=now(), vector_clocks=NewDataVClock}}, NewState }. 45 | 46 | do_order([], []) -> 47 | undefined; 48 | do_order([], UniqList) -> 49 | UniqList; 50 | do_order([Data|Rest], UniqList) -> 51 | VClock = Data#data.vector_clocks, 52 | case lists:any(fun(Other) -> vclock:descends(Other#data.vector_clocks, VClock) end, Rest) of 53 | true -> 54 | do_order(Rest, UniqList); 55 | _ -> 56 | case lists:any(fun(Other) -> vclock:descends(Other#data.vector_clocks, VClock) end, UniqList) of 57 | true -> 58 | do_order(Rest, UniqList); 59 | _ -> 60 | do_order(Rest, [Data|UniqList]) 61 | end 62 | end. 63 | 64 | order(ListOfData, State) when is_list(ListOfData) -> 65 | OrderedData = do_order(ListOfData, []), 66 | {reply, OrderedData, State}; 67 | order(_Other, State) -> 68 | {reply, undefined, State}. 69 | 70 | 71 | %% TODO: raise error if length > 15(=2#1111) 72 | cas_unique(ListOfData) when length(ListOfData) > 2#1111 -> 73 | {error, lists:flatten(io_lib:format("data list is too long (~p)", [length(ListOfData)]))}; 74 | cas_unique(ListOfData) -> 75 | Length = length(ListOfData), 76 | EachBits = trunc(60/Length), 77 | %% TODO: make 128 contant 2008/11/06 by shino 78 | RestBits = 128- EachBits, 79 | cas_unique(lists:map(fun (Data) -> 80 | <> = Data#data.checksum, 81 | CheckSum 82 | end, ListOfData), 83 | EachBits, 84 | Length, 85 | 4). 86 | 87 | cas_unique([], _EachBits, Result, ResultBits) -> 88 | BitsToBePadded = ?CAS_UNIQUE_BITS- ResultBits, 89 | {ok, <>}; 90 | cas_unique([CheckSum | RestCS], EachBits, Result, ResultBits) -> 91 | ResultBits2 = ResultBits + EachBits, 92 | <> = <>, 93 | cas_unique(RestCS, EachBits, Result2, ResultBits2). 94 | 95 | handle_call(stop, _From, State) -> 96 | {stop, normal, stopped, State}; 97 | handle_call({update, Data}, _From, State) -> 98 | update(Data, State); 99 | handle_call({order, ListOfData}, _From, State) -> 100 | order(ListOfData, State). 101 | handle_cast(_Msg, State) -> 102 | {noreply, State}. 103 | handle_info(_Info, State) -> 104 | {noreply, State}. 105 | code_change(_OldVsn, State, _Extra) -> 106 | {ok, State}. 107 | 108 | stop() -> 109 | gen_server:call(?SERVER, stop). 110 | update(Data) -> 111 | gen_server:call(?SERVER, {update, Data}). 112 | order(ListOfData) -> 113 | gen_server:call(?SERVER, {order, ListOfData}). 114 | -------------------------------------------------------------------------------- /src/vclock.erl: -------------------------------------------------------------------------------- 1 | %% @copyright 2007-2008 Basho Technologies 2 | 3 | %% @reference Leslie Lamport (1978). "Time, clocks, and the ordering of events in a distributed system". Communications of the ACM 21 (7): 558-565. 4 | 5 | %% @reference Friedemann Mattern (1988). "Virtual Time and Global States of Distributed Systems". Workshop on Parallel and Distributed Algorithms: pp. 215-226 6 | 7 | %% @author Justin Sheehy 8 | %% @author Andy Gross 9 | 10 | %% @doc A simple Erlang implementation of vector clocks as inspired by Lamport logical clocks. 11 | 12 | %% Copyright 2007-2008 Basho Technologies 13 | 14 | %% Licensed under the Apache License, Version 2.0 (the "License"); 15 | %% you may not use this file except in compliance with the License. 16 | %% You may obtain a copy of the License at 17 | 18 | %% http://www.apache.org/licenses/LICENSE-2.0 19 | 20 | %% Unless required by applicable law or agreed to in writing, software 21 | %% distributed under the License is distributed on an "AS IS" BASIS, 22 | %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 23 | %% See the License for the specific language governing permissions and 24 | %% limitations under the License. 25 | 26 | -module(vclock). 27 | 28 | -author('Justin Sheehy '). 29 | -author('Andy Gross '). 30 | 31 | -export([fresh/0,descends/2,merge/1,get_counter/2,get_timestamp/2, 32 | increment/2,all_nodes/1]). 33 | 34 | %% @type vclock() = [vc_entry]. 35 | %% @type vc_entry() = {node(), {counter(), timestamp()}}. 36 | %% The timestamp is present but not used, in case a client wishes to inspect it. 37 | %% @type node() = term(). 38 | %% Nodes can have any term() as a name, but they must differ from each other. 39 | %% @type counter() = integer(). 40 | %% @type timestamp() = integer(). 41 | 42 | %% @doc Create a brand new vclock. 43 | %% @spec fresh() -> vclock() 44 | fresh() -> 45 | []. 46 | 47 | %% @doc Remove trivial ancestors. 48 | %% @spec simplify(vclock()) -> vclock() 49 | simplify(VClock) -> 50 | simplify(VClock, []). 51 | simplify([], NClock) -> 52 | NClock; 53 | simplify([{Node,{Ctr1,TS1}}|VClock], NClock) -> 54 | {Ctr2,TS2} = proplists:get_value(Node, NClock, {Ctr1,TS1}), 55 | {Ctr,TS} = if 56 | Ctr1 > Ctr2 -> 57 | {Ctr1,TS1}; 58 | true -> 59 | {Ctr2,TS2} 60 | end, 61 | simplify(VClock, [{Node,{Ctr,TS}}|proplists:delete(Node, NClock)]). 62 | 63 | %% @doc Return true if Va is a direct descendant of Vb, else false -- remember, a vclock is its own descendant! 64 | %% @spec descends(Va :: vclock(), Vb :: vclock()) -> bool() 65 | descends(_, []) -> 66 | %% all vclocks descend from the empty vclock 67 | true; 68 | descends(Va, Vb) -> 69 | [{NodeB, {CtrB, _T}}|RestB] = Vb, 70 | CtrA = 71 | case proplists:get_value(NodeB, Va) of 72 | undefined -> 73 | false; 74 | {CA, _TSA} -> CA 75 | end, 76 | case CtrA of 77 | false -> false; 78 | _ -> 79 | if 80 | CtrA < CtrB -> 81 | false; 82 | true -> 83 | descends(Va,RestB) 84 | end 85 | end. 86 | 87 | %% @doc Combine all VClocks in the input list into their least possible 88 | %% common descendant. 89 | %% @spec merge(VClocks :: [vclock()]) -> vclock() 90 | merge(VClocks) -> 91 | merge(VClocks, []). 92 | 93 | merge([], NClock) -> 94 | NClock; 95 | merge([AClock|VClocks],NClock) -> 96 | merge(VClocks, lists:foldl(fun(X,L) -> extend(X,L) end, NClock, AClock)). 97 | 98 | %% @doc Get the counter value in VClock set from Node. 99 | %% @spec get_counter(Node :: node(), VClock :: vclock()) -> counter() 100 | get_counter(Node, VClock) -> 101 | case proplists:get_value(Node, VClock) of 102 | {Ctr, _TS} -> Ctr; 103 | undefined -> undefined 104 | end. 105 | 106 | %% @doc Get the timestamp value in a VClock set from Node. 107 | %% @spec get_timestamp(Node :: node(), VClock :: vclock()) -> timestamp() 108 | get_timestamp(Node, VClock) -> 109 | case proplists:get_value(Node, VClock) of 110 | {_Ctr, TS} -> TS; 111 | undefined -> undefined 112 | end. 113 | 114 | %% @doc Increment VClock at Node. 115 | %% @spec increment(Node :: node(), VClock :: vclock()) -> vclock() 116 | increment(Node, VClock) -> 117 | {Ctr, TS} = case proplists:get_value(Node, VClock) of 118 | undefined -> 119 | {1, timestamp()}; 120 | {C, _T} -> 121 | {C + 1, timestamp()} 122 | end, 123 | extend({Node, {Ctr, TS}}, VClock). 124 | 125 | %% @doc Reflect an update performed by Node. 126 | %% See increment/2 for usage. 127 | %% @spec extend(VC_Entry :: VC_Entry, VClock :: vclock()) -> vclock() 128 | extend({Node,{Ctr,TS}}, VClock) -> 129 | simplify([{Node,{Ctr,TS}}|VClock]). 130 | 131 | %% @doc Return the list of all nodes that have ever incremented VClock. 132 | %% @spec all_nodes(VClock :: vclock()) -> [node()] 133 | all_nodes(VClock) -> 134 | [X || {X,{_,_}} <- VClock]. 135 | 136 | timestamp() -> 137 | calendar:datetime_to_gregorian_seconds(erlang:universaltime()). 138 | -------------------------------------------------------------------------------- /test/kai.config: -------------------------------------------------------------------------------- 1 | [{kai, [ 2 | {logfile, "kai.log"}, 3 | {hostname, "localhost"}, 4 | {rpc_port, 11011}, 5 | {rpc_max_processes, 30}, 6 | {memcache_port, 11211}, 7 | {memcache_max_processes, 10}, 8 | {max_connections, 32}, 9 | {n, 3}, 10 | {r, 2}, 11 | {w, 2}, 12 | {number_of_buckets, 1024}, 13 | {number_of_virtual_nodes, 128}, 14 | {store, dets}, 15 | {dets_dir, "kai"}, 16 | {number_of_tables, 256} 17 | ]}]. 18 | -------------------------------------------------------------------------------- /test/kai.coverspec: -------------------------------------------------------------------------------- 1 | {level, details}. 2 | {incl_mods, [ 3 | kai_config, kai_log, kai_hash, kai_store, kai_stat, kai_version, 4 | kai_connection, kai_sync, kai_membership, kai_coordinator, 5 | kai_tcp_server, 6 | kai_tcp_server_sup, kai_tcp_server_acceptor, kai_tcp_server_monitor, 7 | kai_rpc, kai_memcache 8 | ]}. 9 | 10 | -------------------------------------------------------------------------------- /test/kai_config_SUITE.erl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -module(kai_config_SUITE). 14 | -compile(export_all). 15 | 16 | -include("kai.hrl"). 17 | -include("kai_test.hrl"). 18 | 19 | all() -> [test1]. 20 | 21 | test1() -> []. 22 | test1(_Conf) -> 23 | kai_config:start_link([ 24 | {hostname, "localhost"}, 25 | {rpc_port, 11011}, 26 | {n, 2}, 27 | {number_of_buckets, 16384}, 28 | {number_of_virtual_nodes, 128} 29 | ]), 30 | 31 | ?assertEqual( 32 | ?NODE1, 33 | kai_config:get(node) 34 | ), 35 | ?assertEqual( 36 | 16384, 37 | kai_config:get(number_of_buckets) 38 | ), 39 | ?assertEqual( 40 | 128, 41 | kai_config:get(number_of_virtual_nodes) 42 | ), 43 | 44 | ?assertEqual( 45 | [?NODE1, 16384, 128], 46 | kai_config:get([node, number_of_buckets, number_of_virtual_nodes]) 47 | ), 48 | 49 | ?assertEqual( 50 | {node_info, ?NODE1, [{number_of_virtual_nodes, 128}]}, 51 | kai_config:node_info() 52 | ), 53 | 54 | kai_config:stop(). 55 | -------------------------------------------------------------------------------- /test/kai_connection_SUITE.erl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -module(kai_connection_SUITE). 14 | -compile(export_all). 15 | 16 | -include("kai.hrl"). 17 | -include("kai_test.hrl"). 18 | 19 | all() -> [test1, test2]. 20 | 21 | test1_api_start() -> 22 | {ok, ListeningSocket} = 23 | gen_tcp:listen(11012, [binary, {packet, 4}, {reuseaddr, true}]), 24 | test1_api_accpet(ListeningSocket). 25 | 26 | test1_api_accpet(ListeningSocket) -> 27 | {ok, ApiSocket} = gen_tcp:accept(ListeningSocket), 28 | Pid = spawn(?MODULE, test1_api_proc, [ApiSocket]), 29 | gen_tcp:controlling_process(ApiSocket, Pid), 30 | test1_api_accpet(ListeningSocket). 31 | 32 | test1_api_proc(ApiSocket) -> 33 | receive 34 | {tcp, ApiSocket, _Bin} -> 35 | gen_tcp:send(ApiSocket, term_to_binary(ok)) 36 | end. 37 | 38 | test1_api_send(Pid) -> 39 | {ok, Socket} = kai_connection:lease(?NODE2, self()), 40 | ok = gen_tcp:send(Socket, term_to_binary(ok)), 41 | Pid ! receive {tcp, Socket, Bin} -> binary_to_term(Bin) end. 42 | 43 | test1() -> []. 44 | test1(_Conf) -> 45 | kai_config:start_link([ 46 | {hostname, "localhost"}, 47 | {rpc_port, 11011}, 48 | {max_connections, 32}, 49 | {n, 3}, 50 | {number_of_buckets, 8}, 51 | {number_of_virtual_nodes, 2} 52 | ]), 53 | kai_connection:start_link(), 54 | 55 | spawn_link(?MODULE, test1_api_start, []), 56 | 57 | % lease and return 58 | 59 | {ok, Socket1} = kai_connection:lease(?NODE2, self()), 60 | {ok, Connections} = kai_connection:connections(), 61 | ?assertEqual(1, length(Connections)), 62 | 63 | {ok, Socket2} = kai_connection:lease(?NODE2, self()), 64 | {ok, Connections2} = kai_connection:connections(), 65 | ?assertEqual(2, length(Connections2)), 66 | 67 | ?assert(Socket1 =/= Socket2), 68 | 69 | ok = kai_connection:return(Socket1), 70 | {ok, Connections3} = kai_connection:connections(), 71 | ?assertEqual(2, length(Connections3)), 72 | 73 | {ok, Socket3} = kai_connection:lease(?NODE2, self(), [{active, true}, {packet, 4}]), 74 | 75 | ?assert(Socket1 =:= Socket3), 76 | 77 | ok = kai_connection:close(Socket3), 78 | {ok, Connections4} = kai_connection:connections(), 79 | ?assertEqual(1, length(Connections4)), 80 | 81 | % send and receive at different processes 82 | 83 | spawn_link(?MODULE, test1_api_send, [self()]), 84 | ?assert(receive ok -> true; _ -> false end), 85 | 86 | spawn_link(?MODULE, test1_api_send, [self()]), 87 | ?assert(receive ok -> true; _ -> false end), 88 | 89 | kai_config:stop(), 90 | kai_connection:stop(). 91 | 92 | test2() -> []. 93 | test2(_Conf) -> 94 | MaxConnections = 32, 95 | 96 | kai_config:start_link([ 97 | {hostname, "localhost"}, 98 | {rpc_port, 11011}, 99 | {max_connections, MaxConnections}, 100 | {n, 3}, 101 | {number_of_buckets, 8}, 102 | {number_of_virtual_nodes, 2} 103 | ]), 104 | kai_connection:start_link(), 105 | 106 | spawn_link(?MODULE, test1_api_start, []), 107 | 108 | % lease MaxConnections connections 109 | 110 | Sockets = 111 | lists:map( 112 | fun(_X) -> 113 | {ok, Socket} = kai_connection:lease(?NODE2, self()), 114 | Socket 115 | end, 116 | lists:seq(1, MaxConnections + 1) 117 | ), 118 | 119 | % # of connections can be greater than MaxConnections, because all 120 | % connections are in use 121 | 122 | {ok, Connections} = kai_connection:connections(), 123 | ?assertEqual(MaxConnections + 1, length(Connections)), 124 | 125 | % # of connections equals to MaxConnections, because a connection has 126 | % been returned 127 | 128 | [Socket|_] = Sockets, 129 | ok = kai_connection:return(Socket), 130 | {ok, Connections2} = kai_connection:connections(), 131 | ?assertEqual(MaxConnections, length(Connections2)), 132 | 133 | kai_config:stop(), 134 | kai_connection:stop(). 135 | -------------------------------------------------------------------------------- /test/kai_coordinator_SUITE.erl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -module(kai_coordinator_SUITE). 14 | -compile(export_all). 15 | 16 | -include("kai.hrl"). 17 | -include("kai_test.hrl"). 18 | -include("ct.hrl"). 19 | 20 | all() -> [test1, get_concurrent_data, put_concurrent_data, 21 | put_and_overwrite_stale_data, put_and_fail_overwrite_stale_data]. 22 | 23 | init_per_suite(Config) -> 24 | net_kernel:start([hoge, shortnames]), 25 | {N, R, W} = {2, 2, 2}, 26 | Node1 = start_kai_node(11211, N, R, W), 27 | Config1 = [{node1, Node1} | Config], 28 | Node2 = start_kai_node(11212, N, R, W), 29 | Config2 = [{node2, Node2} | Config1], 30 | {ok, Localhost} = inet:getaddr(localhost, inet), 31 | rpc:call(Node1, kai_membership, check_node, [{Localhost, 11012}]), 32 | rpc:call(Node2, kai_membership, check_node, [{Localhost, 11011}]), 33 | timer:sleep(100), 34 | %% p("nodes at Node1", rpc:call(Node1, kai_hash, node_list, [])), 35 | Config2. 36 | 37 | end_per_suite(Config) -> 38 | Node1 = ?config(node1, Config), 39 | Node2 = ?config(node2, Config), 40 | ok = slave:stop(Node1), 41 | ok = slave:stop(Node2), 42 | ok. 43 | 44 | test1() -> []. 45 | test1(_Conf) -> 46 | kai_config:start_link([ 47 | {hostname, "localhost"}, 48 | {rpc_port, 21011}, 49 | {rpc_max_processes, 2}, 50 | {max_connections, 32}, 51 | {n, 1}, {r, 1}, {w, 1}, 52 | {number_of_buckets, 8}, 53 | {number_of_virtual_nodes, 2}, 54 | {store, ets} 55 | ]), 56 | kai_hash:start_link(), 57 | kai_store:start_link(), 58 | kai_stat:start_link(), 59 | kai_version:start_link(), 60 | kai_connection:start_link(), 61 | kai_rpc:start_link(), 62 | 63 | ?assertEqual( 64 | ok, 65 | kai_coordinator:route({put, #data{ 66 | key = "item-1", 67 | flags = "0", 68 | value = (<<"value-1">>)} 69 | }) 70 | ), 71 | 72 | ListOfData = kai_coordinator:route({get, #data{key="item-1"}}), 73 | ?assertEqual(1, length(ListOfData)), 74 | 75 | [Data|_] = ListOfData, 76 | ?assert(is_record(Data, data)), 77 | ?assertEqual("item-1", Data#data.key), 78 | ?assertEqual(3, Data#data.bucket), 79 | ?assertEqual(erlang:md5(<<"value-1">>), Data#data.checksum), 80 | ?assertEqual("0", Data#data.flags), 81 | ?assertEqual(<<"value-1">>, Data#data.value), 82 | 83 | ?assertEqual( 84 | ok, 85 | kai_coordinator:route({delete, #data{key="item-1"}}) 86 | ), 87 | 88 | ?assertEqual( 89 | undefined, 90 | kai_coordinator:route({get, #data{key="item-1"}}) 91 | ), 92 | 93 | ?assertEqual( 94 | undefined, 95 | kai_coordinator:route({delete, #data{key="item-1"}}) 96 | ), 97 | 98 | kai_rpc:stop(), 99 | kai_connection:stop(), 100 | kai_version:stop(), 101 | kai_stat:stop(), 102 | kai_store:stop(), 103 | kai_hash:stop(), 104 | kai_config:stop(). 105 | 106 | 107 | get_concurrent_data() -> []. 108 | get_concurrent_data(Config) -> 109 | Key = "key1", 110 | Node1 = ?config(node1, Config), 111 | Node2 = ?config(node2, Config), 112 | 113 | ok = rpc:call(Node1, kai_coordinator, route, [{put, #data{key=Key, value="value1"}}]), 114 | [Data] = rpc:call(Node1, kai_coordinator, route, [{get, #data{key=Key}}]), 115 | IntentionalConcurrentVCAtNode1 = vclock:increment(rpc:call(Node1, kai_config, get, [node]), 116 | Data#data.vector_clocks), 117 | IntentionalConcurrentVCAtNode2 = vclock:increment(rpc:call(Node2, kai_config, get, [node]), 118 | Data#data.vector_clocks), 119 | ok = rpc:call(Node1, kai_store, put, [Data#data{vector_clocks=IntentionalConcurrentVCAtNode1}]), 120 | ok = rpc:call(Node2, kai_store, put, [Data#data{vector_clocks=IntentionalConcurrentVCAtNode2}]), 121 | 122 | GetResult = rpc:call(Node1, kai_coordinator, route, [{get, #data{key=Key}}]), 123 | p("get result:", GetResult), 124 | ?assertEqual(2, length(GetResult)), 125 | StatOfUnreconciledGet = 126 | proplists:get_value(kai_unreconciled_get, 127 | rpc:call(Node1, kai_stat, all, [])), 128 | ?assertEqual("1(1)", 129 | lists:sublist(lists:flatten(StatOfUnreconciledGet), 4)), 130 | 131 | ok. 132 | 133 | put_concurrent_data() -> []. 134 | put_concurrent_data(Config) -> 135 | Key = "key2", 136 | Node1 = ?config(node1, Config), 137 | Node2 = ?config(node2, Config), 138 | ok = rpc:call(Node1, kai_coordinator, route, [{put, #data{key=Key, value="value1"}}]), 139 | [Data] = rpc:call(Node1, kai_coordinator, route, [{get, #data{key=Key}}]), 140 | IntentionalConcurrentVCAtNode1 = vclock:increment(rpc:call(Node1, kai_config, get, [node]), 141 | Data#data.vector_clocks), 142 | IntentionalConcurrentVCAtNode2 = vclock:increment(rpc:call(Node2, kai_config, get, [node]), 143 | Data#data.vector_clocks), 144 | ok = rpc:call(Node1, kai_store, put, [Data#data{vector_clocks=IntentionalConcurrentVCAtNode1}]), 145 | ok = rpc:call(Node2, kai_store, put, [Data#data{vector_clocks=IntentionalConcurrentVCAtNode2}]), 146 | {error, Reason} = rpc:call(Node1, 147 | kai_coordinator, route, 148 | [{put, #data{key=Key, 149 | flags = "0", 150 | value = (<<"value-1">>)} 151 | }]), 152 | ?assertEqual(ebusy, Reason), 153 | p("put error reason:", Reason), 154 | ok. 155 | 156 | put_and_overwrite_stale_data() -> []. 157 | put_and_overwrite_stale_data(Config) -> 158 | Key = "key3", 159 | Node1 = ?config(node1, Config), 160 | _Node2 = ?config(node2, Config), 161 | ok = rpc:call(Node1, kai_coordinator, route, [{put, #data{key=Key, value="value1"}}]), 162 | [Data] = rpc:call(Node1, kai_coordinator, route, [{get, #data{key=Key}}]), 163 | IntentionalAhreadVCAtNode1 = vclock:increment(rpc:call(Node1, kai_config, get, [node]), 164 | Data#data.vector_clocks), 165 | ok = rpc:call(Node1, kai_store, put, [Data#data{vector_clocks=IntentionalAhreadVCAtNode1}]), 166 | ok = rpc:call(Node1, 167 | kai_coordinator, route, 168 | [{put, #data{key=Key, 169 | flags = "0", 170 | value = (<<"value-1">>)} 171 | }]), 172 | ok. 173 | 174 | put_and_fail_overwrite_stale_data() -> []. 175 | put_and_fail_overwrite_stale_data(Config) -> 176 | Key = "key3", 177 | Node1 = ?config(node1, Config), 178 | Node2 = ?config(node2, Config), 179 | ok = rpc:call(Node1, kai_coordinator, route, [{put, #data{key=Key, value="value1"}}]), 180 | [Data] = rpc:call(Node1, kai_coordinator, route, [{get, #data{key=Key}}]), 181 | IntentionalAhreadVCAtNode1 = vclock:increment(rpc:call(Node1, kai_config, get, [node]), 182 | Data#data.vector_clocks), 183 | ok = rpc:call(Node1, kai_store, put, [Data#data{vector_clocks=IntentionalAhreadVCAtNode1}]), 184 | %% FIXME: this rpc:call should succeceed. Correct pattern matching is as below: 185 | %% ok = rpc:call(Node2, 186 | {error, Reason} = rpc:call(Node2, 187 | kai_coordinator, route, 188 | [{put, #data{key=Key, 189 | flags = "0", 190 | value = (<<"value-1">>)} 191 | }]), 192 | ?assertEqual(ebusy, Reason), 193 | p("put error reason:", Reason), 194 | ok. 195 | 196 | start_kai_node(MemcachePort, N, R, W) -> 197 | {ok, Node} = slave:start(net_adm:localhost(), 198 | list_to_atom("kai_test_" ++ integer_to_list(MemcachePort)), 199 | "-pa ../../../ebin"), 200 | ok = rpc:call(Node, application, load, [kai]), 201 | p("kai loaded at", Node), 202 | 203 | lists:foreach(fun({Par, Value}) -> 204 | rpc:call(Node, application, set_env, [kai, Par, Value]) 205 | end, [{logfile, "kai_test.log"}, 206 | {hostname, "127.0.0.1"}, 207 | {rpc_port, MemcachePort - 200}, 208 | {rpc_max_processes, 30}, 209 | {memcache_port, MemcachePort}, 210 | {memcache_max_processes, 30}, 211 | {max_connections, 32}, 212 | {n, N}, 213 | {r, R}, 214 | {w, W}, 215 | {number_of_buckets, 1024}, 216 | {number_of_virtual_nodes, 128} 217 | ]), 218 | 219 | ok = rpc:call(Node, application, start, [kai]), 220 | p("kai started at", Node), 221 | Node. 222 | 223 | p(Label, Message) -> 224 | ct:pal(default, "~p: ~p~n", [Label, Message]). 225 | -------------------------------------------------------------------------------- /test/kai_hash_SUITE.erl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -module(kai_hash_SUITE). 14 | -compile(export_all). 15 | 16 | -include("kai.hrl"). 17 | -include("kai_test.hrl"). 18 | 19 | all() -> [test_hash]. % test_hash_efficiency 20 | 21 | % Hash ring of test1: 22 | % 23 | % 0 bucket-0 [3,2,1] -> [3,1,4] 24 | % 98,830,170 item-4 25 | % 26 | % 536,870,912 bucket-1 [3,2,1] -> [3,1,4] 27 | % 953,414,533 item-2 28 | % 29 | % 1,073,741,824 bucket-2 [3,2,1] -> [3,1,4] 30 | % 1,258,937,939 NODE3(2) 31 | % 32 | % 1,610,612,736 bucket-3 [2,1,4] -> [1,4,3] 33 | % 1,704,004,111 item-3 34 | % 1,981,805,867 item-1 35 | % 36 | % 2,147,483,648 bucket-4 [2,1,4] -> [1,4,3] 37 | % 2,203,089,259 NODE2(1) 38 | % 2,311,136,591 NODE1(2) 39 | % 2,365,722,681 NODE4(2) 40 | % 41 | % 2,684,354,560 bucket-5 [2,4,1] -> [4,1,3] 42 | % 2,772,605,746 NODE2(2) 43 | % 2,978,498,268 NODE4(1) 44 | % 45 | % 3,221,225,472 bucket-6 [1,3,2] -> [1,3,4] 46 | % 3,495,790,055 NODE1(1) 47 | % 48 | % 3,758,096,384 bucket-7 [3,2,1] -> [3,1,4] 49 | % 4,264,647,116 NODE3(1) 50 | 51 | test_hash() -> []. 52 | test_hash(_Conf) -> 53 | kai_config:start_link([ 54 | {hostname, "localhost"}, 55 | {rpc_port, 11011}, 56 | {n, 3}, 57 | {number_of_buckets, 8}, 58 | {number_of_virtual_nodes, 2} 59 | ]), 60 | kai_hash:start_link(), 61 | 62 | {node_info, ?NODE1, Info1} = kai_hash:node_info(), 63 | ?assertEqual(?INFO, Info1), 64 | {node_info, ?NODE1, Info1} = kai_hash:node_info(?NODE1), 65 | ?assertEqual(?INFO, Info1), 66 | 67 | {node_list, NodeList1} = kai_hash:node_list(), 68 | ?assertEqual([?NODE1], NodeList1), 69 | 70 | {virtual_node_list, VirtualNodeList1} = kai_hash:virtual_node_list(), 71 | ?assertEqual( 72 | [{2311136591, ?NODE1}, 73 | {3495790055, ?NODE1}], 74 | VirtualNodeList1 75 | ), 76 | 77 | {bucket_list, BucketList1} = kai_hash:bucket_list(), 78 | ?assertEqual( 79 | [{0, [?NODE1]}, 80 | {1, [?NODE1]}, 81 | {2, [?NODE1]}, 82 | {3, [?NODE1]}, 83 | {4, [?NODE1]}, 84 | {5, [?NODE1]}, 85 | {6, [?NODE1]}, 86 | {7, [?NODE1]}], 87 | BucketList1 88 | ), 89 | 90 | {buckets, Buckets1} = kai_hash:buckets(), 91 | ?assertEqual([0,1,2,3,4,5,6,7], Buckets1), 92 | 93 | {replaced_buckets, ReplacedBuckets2} = 94 | kai_hash:update_nodes([{?NODE2, ?INFO}, {?NODE3, ?INFO}, {?NODE4, ?INFO}], 95 | []), 96 | 97 | ?assertEqual( 98 | [{0,3,1}, {1,3,1}, {2,3,1}, {3,2,1}, {4,2,1}, {5,3,1}, {7,3,1}], 99 | ReplacedBuckets2 100 | ), 101 | 102 | {node_list, NodeList2} = kai_hash:node_list(), 103 | ?assertEqual(4, length(NodeList2)), 104 | ?assert(lists:member(?NODE2, NodeList2)), 105 | ?assert(lists:member(?NODE3, NodeList2)), 106 | ?assert(lists:member(?NODE4, NodeList2)), 107 | 108 | {virtual_node_list, VirtualNodeList2} = kai_hash:virtual_node_list(), 109 | ?assertEqual( 110 | [{1258937939, ?NODE3}, 111 | {2203089259, ?NODE2}, 112 | {2311136591, ?NODE1}, 113 | {2365722681, ?NODE4}, 114 | {2772605746, ?NODE2}, 115 | {2978498268, ?NODE4}, 116 | {3495790055, ?NODE1}, 117 | {4264647116, ?NODE3}], 118 | VirtualNodeList2 119 | ), 120 | 121 | {bucket_list, BucketList2} = kai_hash:bucket_list(), 122 | ?assertEqual( 123 | [{0, [?NODE3, ?NODE2, ?NODE1]}, 124 | {1, [?NODE3, ?NODE2, ?NODE1]}, 125 | {2, [?NODE3, ?NODE2, ?NODE1]}, 126 | {3, [?NODE2, ?NODE1, ?NODE4]}, 127 | {4, [?NODE2, ?NODE1, ?NODE4]}, 128 | {5, [?NODE2, ?NODE4, ?NODE1]}, 129 | {6, [?NODE1, ?NODE3, ?NODE2]}, 130 | {7, [?NODE3, ?NODE2, ?NODE1]}], 131 | BucketList2 132 | ), 133 | 134 | {buckets, Buckets2} = kai_hash:buckets(), 135 | ?assertEqual([0,1,2,3,4,5,6,7], Buckets2), 136 | 137 | {bucket, Bucket1} = kai_hash:find_bucket("item-1"), 138 | ?assertEqual(3, Bucket1), 139 | 140 | {replica, Replica1} = kai_hash:find_replica(Bucket1), 141 | ?assertEqual(2, Replica1), 142 | 143 | {nodes, Nodes1} = kai_hash:find_nodes(Bucket1), 144 | ?assertEqual([?NODE2, ?NODE1, ?NODE4], Nodes1), 145 | 146 | {nodes, Nodes2} = kai_hash:find_nodes("item-1"), 147 | ?assertEqual([?NODE2, ?NODE1, ?NODE4], Nodes2), 148 | 149 | {nodes, Nodes3} = kai_hash:find_nodes("item-2"), 150 | ?assertEqual([?NODE3, ?NODE2, ?NODE1], Nodes3), 151 | 152 | {node, Node1} = kai_hash:choose_node_randomly(), 153 | ?assertNot(Node1 == ?NODE1), 154 | 155 | {bucket, Bucket2} = kai_hash:choose_bucket_randomly(), 156 | ?assert((Bucket2 >= 0) or (Bucket2 < 8)), % TODO: choose it from my buckets 157 | 158 | {replaced_buckets, ReplacedBuckets3} = kai_hash:update_nodes([], [?NODE2]), 159 | 160 | ?assertEqual( 161 | [{0,2,3}, {1,2,3}, {2,2,3}, {3,1,2}, {4,1,2}, {5,2,3}, {7,2,3}], 162 | ReplacedBuckets3 163 | ), 164 | 165 | {node_list, NodeList3} = kai_hash:node_list(), 166 | ?assertEqual(3, length(NodeList3)), 167 | ?assertNot(lists:member(?NODE2, NodeList3)), 168 | 169 | {virtual_node_list, VirtualNodeList3} = kai_hash:virtual_node_list(), 170 | ?assertEqual( 171 | [{1258937939, ?NODE3}, 172 | {2311136591, ?NODE1}, 173 | {2365722681, ?NODE4}, 174 | {2978498268, ?NODE4}, 175 | {3495790055, ?NODE1}, 176 | {4264647116, ?NODE3}], 177 | VirtualNodeList3 178 | ), 179 | 180 | {bucket_list, BucketList3} = kai_hash:bucket_list(), 181 | ?assertEqual( 182 | [{0, [?NODE3, ?NODE1, ?NODE4]}, 183 | {1, [?NODE3, ?NODE1, ?NODE4]}, 184 | {2, [?NODE3, ?NODE1, ?NODE4]}, 185 | {3, [?NODE1, ?NODE4, ?NODE3]}, 186 | {4, [?NODE1, ?NODE4, ?NODE3]}, 187 | {5, [?NODE4, ?NODE1, ?NODE3]}, 188 | {6, [?NODE1, ?NODE3, ?NODE4]}, 189 | {7, [?NODE3, ?NODE1, ?NODE4]}], 190 | BucketList3 191 | ), 192 | 193 | {buckets, Buckets3} = kai_hash:buckets(), 194 | ?assertEqual([0,1,2,3,4,5,6,7], Buckets3), 195 | 196 | {nodes, Nodes4} = kai_hash:find_nodes("item-1"), 197 | ?assertEqual([?NODE1, ?NODE4, ?NODE3], Nodes4), 198 | 199 | {nodes, Nodes5} = kai_hash:find_nodes("item-2"), 200 | ?assertEqual([?NODE3, ?NODE1, ?NODE4], Nodes5), 201 | 202 | kai_hash:stop(), 203 | kai_config:stop(). 204 | 205 | test_hash_efficiency() -> []. 206 | test_hash_efficiency(_Conf) -> 207 | io:format("simulate network of 64 nodes"), 208 | 209 | kai_config:start_link([ 210 | {hostname, "localhost"}, 211 | {rpc_port, 1}, 212 | {n, 3}, 213 | {number_of_buckets, 16384}, % 16,384 = 128*64*2 214 | {number_of_virtual_nodes, 128} 215 | ]), 216 | kai_hash:start_link(), 217 | 218 | Nodes = 219 | lists:map( 220 | fun(Port) -> {{{127,0,0,1}, Port}, ?INFO} end, 221 | lists:seq(2, 63) 222 | ), 223 | kai_hash:update_nodes(Nodes, []), 224 | 225 | Args = [[{{{127,0,0,1}, 64}, [{number_of_virtual_nodes, 128}]}], []], 226 | {Usec, _} = timer:tc(kai_hash, update, Args), 227 | io:format("time to add a node: ~p [usec]", [Usec]), 228 | ?assert(Usec < 300000), 229 | 230 | {Usec2, _} = timer:tc(kai_hash, find, ["item-1", 1]), 231 | io:format("time to find a node: ~p [usec]", [Usec2]), 232 | ?assert(Usec2 < 1000), 233 | 234 | {Usec3, _} = timer:tc(kai_hash, choose_node_randomly, []), 235 | io:format("time to choose a node randomly: ~p [usec]", [Usec3]), 236 | ?assert(Usec3 < 1000), 237 | 238 | {Usec4, _} = timer:tc(kai_hash, choose_bucket_randomly, []), 239 | io:format("time to choose a bucket randomly: ~p [usec]", [Usec4]), 240 | ?assert(Usec4 < 300000), 241 | 242 | {Usec5, _} = timer:tc(kai_hash, update, [[], [{{127,0,0,1}, 1}]]), 243 | io:format("time to remove a node: ~p [usec]", [Usec5]), 244 | ?assert(Usec5 < 300000), 245 | 246 | kai_hash:stop(), 247 | kai_config:stop(). 248 | -------------------------------------------------------------------------------- /test/kai_log_SUITE.erl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -module(kai_log_SUITE). 14 | -compile(export_all). 15 | 16 | -include("kai.hrl"). 17 | -include("kai_test.hrl"). 18 | 19 | all() -> [test1]. 20 | 21 | test1() -> []. 22 | test1(_Conf) -> 23 | kai_config:start_link([ 24 | {hostname, "localhost"}, 25 | {rpc_port, 11011}, 26 | {n, 2}, 27 | {number_of_buckets, 16384}, % 16,384 = 128*64*2 28 | {number_of_virtual_nodes, 128} 29 | ]), 30 | kai_log:start_link(), 31 | 32 | ?info("This is information log"), 33 | ?debug(io_lib:format("This is debug log: ~s", ["Hello, World!"])), 34 | 35 | kai_log:stop(), 36 | kai_config:stop(). 37 | -------------------------------------------------------------------------------- /test/kai_membership_SUITE.erl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -module(kai_membership_SUITE). 14 | -compile(export_all). 15 | 16 | -include("kai.hrl"). 17 | -include("kai_test.hrl"). 18 | 19 | all() -> [test1]. 20 | 21 | test1_api_start() -> 22 | {ok, ListeningSocket} = 23 | gen_tcp:listen(11012, [binary, {packet, 4}, {reuseaddr, true}]), 24 | test1_api_accpet(ListeningSocket). 25 | 26 | test1_api_accpet(ListeningSocket) -> 27 | {ok, ApiSocket} = gen_tcp:accept(ListeningSocket), 28 | Pid = spawn(?MODULE, test1_api_proc, [ApiSocket]), 29 | gen_tcp:controlling_process(ApiSocket, Pid), 30 | test1_api_accpet(ListeningSocket). 31 | 32 | test1_api_proc(ApiSocket) -> 33 | receive 34 | {tcp, ApiSocket, Bin} -> 35 | test1_api_send(ApiSocket, binary_to_term(Bin)), 36 | test1_api_proc(ApiSocket) 37 | end. 38 | 39 | test1_api_send(ApiSocket, node_info) -> 40 | NodeInfo = {node_info, ?NODE2, ?INFO}, 41 | gen_tcp:send(ApiSocket, term_to_binary(NodeInfo)); 42 | test1_api_send(ApiSocket, node_list) -> 43 | NodeList = [?NODE1, ?NODE2, ?NODE3, ?NODE4], 44 | gen_tcp:send(ApiSocket, term_to_binary({node_list, NodeList})); 45 | test1_api_send(ApiSocket, {list, 3 = _Bucket}) -> 46 | ListOfData = [#data{ 47 | key = ("item-1"), 48 | bucket = 3, 49 | last_modified = now(), 50 | checksum = erlang:md5(<<"item-1">>) 51 | }], 52 | gen_tcp:send(ApiSocket, term_to_binary({list_of_data, ListOfData})); 53 | test1_api_send(ApiSocket, {list, _Bucket}) -> 54 | gen_tcp:send(ApiSocket, term_to_binary({list_of_data, []})); 55 | test1_api_send(ApiSocket, {get, #data{key="item-1", bucket=3}}) -> 56 | Data = #data{ 57 | key = "item-1", 58 | bucket = 3, 59 | last_modified = now(), 60 | checksum = erlang:md5(<<"value-1">>), 61 | flags = "0", 62 | value = (<<"value-1">>) 63 | }, 64 | gen_tcp:send(ApiSocket, term_to_binary(Data)). 65 | 66 | test1() -> []. 67 | test1(_Conf) -> 68 | % This is NODE3, not NODE1 69 | kai_config:start_link([ 70 | {hostname, "localhost"}, 71 | {rpc_port, 11013}, 72 | {n, 3}, 73 | {number_of_buckets, 8}, 74 | {number_of_virtual_nodes, 2}, 75 | {store, ets} 76 | ]), 77 | kai_hash:start_link(), 78 | kai_store:start_link(), 79 | kai_connection:start_link(), 80 | kai_sync:start_link(), 81 | kai_membership:start_link(), 82 | 83 | {replaced_buckets, _ReplacedBuckets} 84 | = kai_hash:update_nodes([{?NODE1, ?INFO}, {?NODE4, ?INFO}], []), 85 | 86 | Data1 = #data{ 87 | key = ("item-1"), 88 | bucket = 3, 89 | last_modified = now(), 90 | checksum = erlang:md5(<<"value-1">>), flags="0", value=(<<"value-1">>) 91 | }, 92 | kai_store:put(Data1), 93 | 94 | spawn_link(?MODULE, test1_api_start, []), 95 | 96 | kai_membership:check_node(?NODE2), 97 | 98 | timer:sleep(100), 99 | 100 | {node_list, NodeList} = kai_hash:node_list(), 101 | ?assertEqual(4, length(NodeList)), 102 | ?assert(lists:member(?NODE2, NodeList)), 103 | 104 | {list_of_data, ListOfData} = kai_store:list(3), 105 | ?assertEqual(0, length(ListOfData)), 106 | 107 | kai_membership:check_node(?NODE1), 108 | 109 | timer:sleep(100), 110 | 111 | {node_list, NodeList2} = kai_hash:node_list(), 112 | ?assertEqual(3, length(NodeList2)), 113 | ?assertNot(lists:member(?NODE1, NodeList2)), 114 | 115 | {list_of_data, ListOfData2} = kai_store:list(3), 116 | ?assertEqual(1, length(ListOfData2)), 117 | ?assert(lists:keymember("item-1", 2, ListOfData2)), 118 | 119 | timer:sleep(2100), % timeout and check ?NODE4 by kai_hash:choose_node_randomly/0 120 | 121 | {node_list, NodeList3} = kai_hash:node_list(), 122 | ?assertEqual(2, length(NodeList3)), 123 | ?assertNot(lists:member(?NODE4, NodeList3)), 124 | 125 | kai_membership:stop(), 126 | kai_sync:stop(), 127 | kai_connection:stop(), 128 | kai_store:stop(), 129 | kai_hash:stop(), 130 | kai_config:stop(). 131 | -------------------------------------------------------------------------------- /test/kai_memcache_SUITE.erl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -module(kai_memcache_SUITE). 14 | -compile(export_all). 15 | 16 | -include("ct.hrl"). 17 | -include("kai.hrl"). 18 | -include("kai_test.hrl"). 19 | 20 | all() -> [test1]. 21 | 22 | test1() -> []. 23 | test1(_Conf) -> 24 | kai_config:start_link([ 25 | {hostname, "localhost"}, 26 | {rpc_port, 11011}, 27 | {rpc_max_processes, 2}, 28 | {memcache_port, 11211}, 29 | {memcache_max_processes, 2}, 30 | {max_connections, 32}, 31 | {n, 1}, {r, 1}, {w, 1}, 32 | {number_of_buckets, 8}, 33 | {number_of_virtual_nodes, 2}, 34 | {store, ets} 35 | ]), 36 | kai_hash:start_link(), 37 | kai_store:start_link(), 38 | kai_stat:start_link(), 39 | kai_version:start_link(), 40 | kai_connection:start_link(), 41 | kai_rpc:start_link(), 42 | kai_memcache:start_link(), 43 | 44 | timer:sleep(100), % wait for starting kai_memcache 45 | 46 | {ok, MemcacheSocket} = 47 | gen_tcp:connect({127,0,0,1}, 11211, [binary, {packet, line}, {active, false}]), 48 | 49 | Value = <<"value-1">>, 50 | Buf = io_lib:format("set item-1 0 0 ~w\r\n~s\r\n", [byte_size(Value), Value]), 51 | gen_tcp:send(MemcacheSocket, Buf), 52 | 53 | ?assertEqual( 54 | {ok, <<"STORED\r\n">>}, 55 | gen_tcp:recv(MemcacheSocket, 0) 56 | ), 57 | 58 | gen_tcp:send(MemcacheSocket, "get item-1\r\n"), 59 | 60 | ?assertEqual( 61 | {ok, <<"VALUE item-1 0 7\r\n">>}, 62 | gen_tcp:recv(MemcacheSocket, 0) 63 | ), 64 | inet:setopts(MemcacheSocket, [{packet, raw}]), 65 | ?assertEqual( 66 | {ok, <<"value-1">>}, 67 | gen_tcp:recv(MemcacheSocket, byte_size(<<"value-1">>)) 68 | ), 69 | inet:setopts(MemcacheSocket, [{packet, line}]), 70 | ?assertEqual( 71 | {ok, <<"\r\n">>}, 72 | gen_tcp:recv(MemcacheSocket, 0) 73 | ), 74 | ?assertEqual( 75 | {ok, <<"END\r\n">>}, 76 | gen_tcp:recv(MemcacheSocket, 0) 77 | ), 78 | 79 | gen_tcp:send(MemcacheSocket, "delete item-1\r\n"), 80 | ?assertEqual( 81 | {ok, <<"DELETED\r\n">>}, 82 | gen_tcp:recv(MemcacheSocket, 0) 83 | ), 84 | 85 | gen_tcp:send(MemcacheSocket, "delete item-2 0\r\n"), 86 | ?assertEqual( 87 | {ok, <<"NOT_FOUND\r\n">>}, 88 | gen_tcp:recv(MemcacheSocket, 0) 89 | ), 90 | 91 | gen_tcp:send(MemcacheSocket, "get item-1\r\n"), 92 | ?assertEqual( 93 | {ok, <<"END\r\n">>}, 94 | gen_tcp:recv(MemcacheSocket, 0) 95 | ), 96 | 97 | gen_tcp:send(MemcacheSocket, "no_such_command\r\n"), 98 | ?assertEqual( 99 | {ok, <<"ERROR\r\n">>}, 100 | gen_tcp:recv(MemcacheSocket, 0) 101 | ), 102 | 103 | gen_tcp:send(MemcacheSocket, "stats\r\n"), 104 | {ok, Uptime } = gen_tcp:recv(MemcacheSocket, 0), 105 | {ok, Time } = gen_tcp:recv(MemcacheSocket, 0), 106 | {ok, Version } = gen_tcp:recv(MemcacheSocket, 0), 107 | {ok, Bytes } = gen_tcp:recv(MemcacheSocket, 0), 108 | {ok, CurrItems } = gen_tcp:recv(MemcacheSocket, 0), 109 | {ok, CurrConnections } = gen_tcp:recv(MemcacheSocket, 0), 110 | {ok, CmdGet } = gen_tcp:recv(MemcacheSocket, 0), 111 | {ok, CmdSet } = gen_tcp:recv(MemcacheSocket, 0), 112 | {ok, BytesRead } = gen_tcp:recv(MemcacheSocket, 0), 113 | {ok, BytesWrite } = gen_tcp:recv(MemcacheSocket, 0), 114 | {ok, LocalNode } = gen_tcp:recv(MemcacheSocket, 0), 115 | {ok, Quorum } = gen_tcp:recv(MemcacheSocket, 0), 116 | {ok, NumberOfBuckets } = gen_tcp:recv(MemcacheSocket, 0), 117 | {ok, NumberOfVirtualNodes} = gen_tcp:recv(MemcacheSocket, 0), 118 | {ok, Store } = gen_tcp:recv(MemcacheSocket, 0), 119 | {ok, Nodes } = gen_tcp:recv(MemcacheSocket, 0), 120 | {ok, UnreconciledGet } = gen_tcp:recv(MemcacheSocket, 0), 121 | % {ok, Buckets } = gen_tcp:recv(MemcacheSocket, 0), 122 | {ok, ErlangProcs } = gen_tcp:recv(MemcacheSocket, 0), 123 | {ok, ErlangVersion } = gen_tcp:recv(MemcacheSocket, 0), 124 | {match, _S1, _L1} = regexp:match(binary_to_list(Uptime), "uptime"), 125 | {match, _S2, _L2} = regexp:match(binary_to_list(Time), "time"), 126 | {match, _S3, _L3} = regexp:match(binary_to_list(Version), "version"), 127 | {match, _S4, _L4} = regexp:match(binary_to_list(Bytes), "bytes"), 128 | {match, _S5, _L5} = regexp:match(binary_to_list(CurrItems), "curr_items"), 129 | {match, _S6, _L6} = regexp:match(binary_to_list(CurrConnections), "curr_connections"), 130 | {match, _S7, _L7} = regexp:match(binary_to_list(CmdGet), "cmd_get"), 131 | {match, _S8, _L8} = regexp:match(binary_to_list(CmdSet), "cmd_set"), 132 | {match, _S9, _L9} = regexp:match(binary_to_list(BytesRead), "bytes_read"), 133 | {match, _S0, _L0} = regexp:match(binary_to_list(BytesWrite), "bytes_write"), 134 | {match, _Sa, _La} = regexp:match(binary_to_list(LocalNode), "kai_node"), 135 | {match, _Sb, _Lb} = regexp:match(binary_to_list(Quorum), "kai_quorum"), 136 | {match, _Sc, _Lc} = regexp:match(binary_to_list(NumberOfBuckets), "kai_number_of_buckets"), 137 | {match, _Sd, _Ld} = regexp:match(binary_to_list(NumberOfVirtualNodes), "kai_number_of_virtual_nodes"), 138 | {match, _Se, _Le} = regexp:match(binary_to_list(Store), "kai_store"), 139 | {match, _Sf, _Lf} = regexp:match(binary_to_list(Nodes), "kai_curr_nodes"), 140 | {match, _Sk, _Lk} = regexp:match(binary_to_list(UnreconciledGet), "kai_unreconciled_get"), 141 | % {match, _Sg, _Lg} = regexp:match(binary_to_list(Buckets), "kai_curr_buckets"), 142 | {match, _Sh, _Lh} = regexp:match(binary_to_list(ErlangProcs), "erlang_procs"), 143 | {match, _Si, _Li} = regexp:match(binary_to_list(ErlangVersion), "erlang_version"), 144 | ?assertEqual( 145 | {ok, <<"END\r\n">>}, 146 | gen_tcp:recv(MemcacheSocket, 0) 147 | ), 148 | 149 | gen_tcp:send(MemcacheSocket, "version\r\n"), 150 | {ok, Version2} = gen_tcp:recv(MemcacheSocket, 0), 151 | {match, _Sj, _Lj} = regexp:match(binary_to_list(Version2), "VERSION "), 152 | 153 | gen_tcp:send(MemcacheSocket, "quit\r\n"), 154 | 155 | gen_tcp:close(MemcacheSocket), 156 | 157 | kai_memcache:stop(), 158 | kai_rpc:stop(), 159 | kai_connection:stop(), 160 | kai_version:stop(), 161 | kai_stat:stop(), 162 | kai_store:stop(), 163 | kai_hash:stop(), 164 | kai_config:stop(). 165 | -------------------------------------------------------------------------------- /test/kai_rpc_SUITE.erl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -module(kai_rpc_SUITE). 14 | -compile(export_all). 15 | 16 | -include("kai.hrl"). 17 | -include("kai_test.hrl"). 18 | 19 | all() -> [test1]. 20 | 21 | test1() -> []. 22 | test1(_Conf) -> 23 | kai_config:start_link([ 24 | {hostname, "localhost"}, 25 | {rpc_port, 11011}, 26 | {rpc_max_processes, 2}, 27 | {n, 3}, 28 | {number_of_buckets, 8}, 29 | {number_of_virtual_nodes, 2}, 30 | {store, ets} 31 | ]), 32 | kai_hash:start_link(), 33 | kai_store:start_link(), 34 | kai_connection:start_link(), 35 | kai_rpc:start_link(), 36 | 37 | timer:sleep(100), % wait for starting kai_rpc 38 | 39 | {node_info, ?NODE1, ?INFO} = kai_rpc:node_info(?NODE1), 40 | 41 | {node_list, [?NODE1]} = kai_rpc:node_list(?NODE1), 42 | 43 | Data = #data{ 44 | key = "item-1", 45 | bucket = 3, 46 | last_modified = now(), 47 | checksum = erlang:md5(<<"value-1">>), 48 | flags = "0", 49 | value = (<<"value-1">>) 50 | }, 51 | ok = kai_rpc:put(?NODE1, Data), 52 | ?assertEqual(Data, kai_store:get(#data{key="item-1", bucket=3})), 53 | 54 | ListOfData = #data{ 55 | key = "item-1", 56 | bucket = 3, 57 | last_modified = Data#data.last_modified, 58 | checksum = erlang:md5(<<"value-1">>) 59 | }, 60 | {list_of_data, [ListOfData]} = kai_rpc:list(?NODE1, 3), 61 | 62 | Data = kai_rpc:get(?NODE1, #data{key="item-1", bucket=3}), 63 | 64 | ok = kai_rpc:delete(?NODE1, #data{key="item-1", bucket=3}), 65 | 66 | undefined = kai_rpc:get(?NODE1, #data{key="item-1", bucket=3}), 67 | 68 | kai_rpc:stop(), 69 | kai_connection:stop(), 70 | kai_store:stop(), 71 | kai_hash:stop(), 72 | kai_config:stop(). 73 | -------------------------------------------------------------------------------- /test/kai_stat_SUITE.erl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -module(kai_stat_SUITE). 14 | -compile(export_all). 15 | 16 | -include("kai.hrl"). 17 | -include("kai_test.hrl"). 18 | 19 | all() -> [test_all]. 20 | 21 | test_all() -> []. 22 | test_all(_Conf) -> 23 | kai_config:start_link([ 24 | {hostname, "localhost"}, 25 | {rpc_port, 11011}, 26 | {memcache_port, 11211}, 27 | {memcache_max_processes, 1}, 28 | {n, 3}, {r, 2}, {w, 2}, 29 | {number_of_buckets, 8}, 30 | {number_of_virtual_nodes, 2}, 31 | {store, ets} 32 | ]), 33 | kai_hash:start_link(), 34 | kai_store:start_link(), 35 | kai_stat:start_link(), 36 | kai_memcache:start_link(), 37 | 38 | kai_stat:incr_cmd_get(), 39 | kai_stat:incr_cmd_set(), 40 | 41 | Data = #data{value = (<<"value-1">>)}, 42 | kai_stat:add_bytes_read([Data, Data]), 43 | kai_stat:add_bytes_write(Data), 44 | lists:map(fun(Arg) -> 45 | kai_stat:incr_unreconciled_get(Arg) 46 | end, 47 | [{2, true},{2, true},{2, false},{3,true},{5,false}]), 48 | 49 | [{uptime, Uptime }, 50 | {time, Time }, 51 | {version, Version }, 52 | {bytes, Bytes }, 53 | {curr_items, CurrItems }, 54 | {curr_connections, CurrConnections }, 55 | {cmd_get, CmdGet }, 56 | {cmd_set, CmdSet }, 57 | {bytes_read, BytesRead }, 58 | {bytes_write, BytesWrite }, 59 | {kai_node, LocalNode }, 60 | {kai_quorum, Quorum }, 61 | {kai_number_of_buckets, NumberOfBuckets }, 62 | {kai_number_of_virtual_nodes, NumberOfVirtualNodes}, 63 | {kai_store, Store }, 64 | {kai_curr_nodes, Nodes }, 65 | {kai_unreconciled_get, UnreconciledGet }, 66 | % {kai_curr_buckets, Buckets }, 67 | {erlang_procs, ErlangProcs }, 68 | {erlang_version, ErlangVersion }] = kai_stat:all(), 69 | {match, _S1, _L1} = regexp:match(Uptime, "[0-9]+"), 70 | {match, _S2, _L2} = regexp:match(Time, "[0-9]+"), 71 | {match, _S3, _L3} = regexp:match(Version, "[.0-9]+"), 72 | {match, _S4, _L4} = regexp:match(Bytes, "[0-9]+"), 73 | ?assertEqual("0", CurrItems ), 74 | ?assertEqual("0", CurrConnections ), 75 | ?assertEqual("1", CmdGet ), 76 | ?assertEqual("1", CmdSet ), 77 | ?assertEqual("14", BytesRead ), 78 | ?assertEqual("7", BytesWrite ), 79 | ?assertEqual("127.0.0.1:11011", LocalNode ), 80 | ?assertEqual("3,2,2", Quorum ), 81 | ?assertEqual("8", NumberOfBuckets ), 82 | ?assertEqual("2", NumberOfVirtualNodes), 83 | ?assertEqual("ets", Store ), 84 | ?assertEqual("127.0.0.1:11011", Nodes ), 85 | ?assertEqual("1(3) 0(1) 0(0) 1(1)", lists:flatten(UnreconciledGet)), 86 | % ?assertEqual("0 1 2 3 4 5 6 7", Buckets ), 87 | {match, _S5, _L5} = regexp:match(ErlangProcs, "[0-9]+"), 88 | {match, _S6, _L6} = regexp:match(ErlangVersion, "[.0-9]+"), 89 | 90 | kai_memcache:stop(), 91 | kai_stat:stop(), 92 | kai_store:stop(), 93 | kai_hash:stop(), 94 | kai_config:stop(). 95 | -------------------------------------------------------------------------------- /test/kai_store_SUITE.erl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -module(kai_store_SUITE). 14 | -compile(export_all). 15 | 16 | -include("kai.hrl"). 17 | -include("kai_test.hrl"). 18 | 19 | all() -> [test_ets, test_dets, 20 | test_update_conflict_ets, test_update_conflict_dets, 21 | test_perf]. 22 | 23 | test(Conf) -> 24 | kai_config:start_link(Conf), 25 | kai_config:start_link([ 26 | {hostname, "localhost"}, 27 | {rpc_port, 11511}, 28 | {rpc_max_processes, 2}, 29 | {max_connections, 32}, 30 | {n, 1}, {r, 1}, {w, 1}, 31 | {number_of_buckets, 8}, 32 | {number_of_virtual_nodes, 2} 33 | ]), 34 | kai_version:start_link(), 35 | kai_store:start_link(), 36 | 37 | Data1 = #data{ 38 | key = "item-1", 39 | bucket = 3, 40 | last_modified = now(), 41 | checksum = erlang:md5(<<"value-1">>), 42 | flags = "0", 43 | vector_clocks = vclock:fresh(), 44 | value = (<<"value-1">>) 45 | }, 46 | 47 | kai_store:put(Data1), 48 | 49 | ?assertEqual( 50 | Data1, 51 | kai_store:get(#data{key="item-1", bucket=3}) 52 | ), 53 | ?assertEqual( 54 | undefined, 55 | kai_store:get(#data{key="item-2", bucket=1}) 56 | ), 57 | 58 | Data2 = #data{ 59 | key = "item-2", 60 | bucket = 1, 61 | last_modified = now(), 62 | checksum = erlang:md5(<<"value-2">>), 63 | flags = "0", 64 | vector_clocks = [], 65 | value = (<<"value-2">>) 66 | }, 67 | kai_store:put(Data2), 68 | ?assertEqual( 69 | Data2, 70 | kai_store:get(#data{key="item-2", bucket=1}) 71 | ), 72 | 73 | Data3 = #data{ 74 | key = "item-3", 75 | bucket = 3, 76 | last_modified = now(), 77 | checksum = erlang:md5(<<"value-3">>), 78 | flags = "0", 79 | vector_clocks = [], 80 | value = (<<"value-3">>) 81 | }, 82 | kai_store:put(Data3), 83 | ?assertEqual( 84 | Data3, 85 | kai_store:get(#data{key="item-3", bucket=3}) 86 | ), 87 | 88 | {list_of_data, ListOfData1} = kai_store:list(1), 89 | ?assertEqual(1, length(ListOfData1)), 90 | ?assert(lists:keymember("item-2", 2, ListOfData1)), 91 | 92 | {list_of_data, ListOfData2} = kai_store:list(2), 93 | ?assertEqual(0, length(ListOfData2)), 94 | 95 | {list_of_data, ListOfData3} = kai_store:list(3), 96 | ?assertEqual(2, length(ListOfData3)), 97 | ?assert(lists:keymember("item-1", 2, ListOfData3)), 98 | ?assert(lists:keymember("item-3", 2, ListOfData3)), 99 | 100 | Data1b = #data{ 101 | key = "item-1", 102 | bucket = 3, 103 | last_modified = now(), 104 | checksum = erlang:md5(<<"value-1">>), 105 | flags = "0", 106 | vector_clocks = [], 107 | value = (<<"value-1b">>) 108 | }, 109 | 110 | kai_store:put(Data1b), 111 | ?assertEqual( 112 | Data1b, 113 | kai_store:get(#data{key="item-1", bucket=3}) 114 | ), 115 | 116 | kai_store:delete(#data{key="item-1", bucket=3}), 117 | 118 | ?assertEqual( 119 | undefined, 120 | kai_store:get(#data{key="item-1", bucket=3}) 121 | ), 122 | 123 | {list_of_data, ListOfData4} = kai_store:list(3), 124 | ?assertEqual(1, length(ListOfData4)), 125 | ?assert(lists:keymember("item-3", 2, ListOfData4)), 126 | 127 | ?assert(is_integer(kai_store:info(bytes))), 128 | ?assertEqual(2, kai_store:info(size)), 129 | 130 | kai_store:stop(), 131 | kai_version:stop(), 132 | kai_config:stop(), 133 | 134 | ok. 135 | 136 | test_ets() -> []. 137 | test_ets(_Conf) -> 138 | test([ 139 | {hostname, "localhost"}, 140 | {rpc_port, 11011}, 141 | {n, 3}, 142 | {number_of_buckets, 8}, 143 | {number_of_virtual_nodes, 2}, 144 | {store, ets} 145 | ]). 146 | 147 | test_dets() -> []. 148 | test_dets(_Conf) -> 149 | test([ 150 | {hostname, "localhost"}, 151 | {rpc_port, 11011}, 152 | {n, 3}, 153 | {number_of_buckets, 8}, 154 | {number_of_virtual_nodes, 2}, 155 | {store, dets}, 156 | {dets_dir, "."}, 157 | {number_of_tables, 2} 158 | ]), 159 | file:delete("./1"), file:delete("./2"). 160 | 161 | test_update_conflict(Conf) -> 162 | kai_config:start_link(Conf), 163 | kai_config:start_link([ 164 | {hostname, "localhost"}, 165 | {rpc_port, 11511}, 166 | {rpc_max_processes, 2}, 167 | {max_connections, 32}, 168 | {n, 1}, {r, 1}, {w, 1}, 169 | {number_of_buckets, 8}, 170 | {number_of_virtual_nodes, 2} 171 | ]), 172 | kai_version:start_link(), 173 | kai_store:start_link(), 174 | 175 | Data1 = #data{ 176 | key = "item-1", 177 | bucket = 3, 178 | last_modified = now(), 179 | checksum = erlang:md5(<<"value-1">>), 180 | flags = "0", 181 | vector_clocks = vclock:increment(node1, vclock:fresh()), 182 | value = (<<"value-1">>) 183 | }, 184 | kai_store:put(Data1), 185 | 186 | %% conflict vclock 187 | Data2 = Data1#data{ 188 | vector_clocks = vclock:increment(node2, vclock:fresh()) 189 | }, 190 | 191 | {error, Reason} = kai_store:put(Data2), 192 | ct:pal(default, "~p: ~p~n", ["Reason", Reason]), 193 | 194 | kai_store:stop(), 195 | kai_version:stop(), 196 | kai_config:stop(), 197 | 198 | ok. 199 | 200 | test_update_conflict_ets() -> []. 201 | test_update_conflict_ets(_Conf) -> 202 | test_update_conflict([ 203 | {hostname, "localhost"}, 204 | {rpc_port, 11011}, 205 | {n, 3}, 206 | {number_of_buckets, 8}, 207 | {number_of_virtual_nodes, 2}, 208 | {store, ets} 209 | ]). 210 | 211 | test_update_conflict_dets() -> []. 212 | test_update_conflict_dets(_Conf) -> 213 | test_update_conflict([ 214 | {hostname, "localhost"}, 215 | {rpc_port, 11011}, 216 | {n, 3}, 217 | {number_of_buckets, 8}, 218 | {number_of_virtual_nodes, 2}, 219 | {store, dets}, 220 | {dets_dir, "."}, 221 | {number_of_tables, 2} 222 | ]), 223 | file:delete("./1"), file:delete("./2"). 224 | 225 | test_perf_put(T) -> 226 | lists:foreach( 227 | fun(I) -> 228 | Key = "item-" ++ integer_to_list(I), 229 | Value = list_to_binary("value-" ++ integer_to_list(I)), 230 | Data = #data{ 231 | key = Key, 232 | bucket = 3, 233 | last_modified = now(), 234 | checksum = erlang:md5(Value), 235 | flags = "0", 236 | vector_clocks = [], 237 | value = Value 238 | }, 239 | kai_store:put(Data) 240 | end, 241 | lists:seq(1, T) 242 | ). 243 | 244 | test_perf_get(T) -> 245 | lists:foreach( 246 | fun(I) -> 247 | Key = "item-" ++ integer_to_list(I), 248 | kai_store:get(#data{key=Key, bucket=0}) 249 | end, 250 | lists:seq(1, T) 251 | ). 252 | 253 | test_perf() -> []. 254 | test_perf(_Conf) -> 255 | kai_config:start_link([ 256 | {hostname, "localhost"}, 257 | {rpc_port, 11011}, 258 | {n, 3}, 259 | {number_of_buckets, 8}, 260 | {number_of_virtual_nodes, 2}, 261 | {store, ets} 262 | ]), 263 | kai_store:start_link(), 264 | 265 | T = 10000, 266 | 267 | {Usec, _} = timer:tc(?MODULE, test_perf_put, [T]), 268 | ?assert(Usec < 100*T), 269 | io:format("average time to put data: ~p [usec]", [Usec/T]), 270 | 271 | {Usec2, _} = timer:tc(?MODULE, test_perf_get, [T]), 272 | ?assert(Usec2 < 100*T), 273 | io:format("average time to get data: ~p [usec]", [Usec2/T]), 274 | kai_store:stop(), 275 | kai_config:stop(). 276 | 277 | p(Label, Message) -> 278 | ct:pal(default, "~p: ~p~n", [Label, Message]). 279 | -------------------------------------------------------------------------------- /test/kai_sync_SUITE.erl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -module(kai_sync_SUITE). 14 | -compile(export_all). 15 | 16 | -include("kai.hrl"). 17 | -include("kai_test.hrl"). 18 | 19 | all() -> [test1]. 20 | 21 | test1_api_start() -> 22 | {ok, ListeningSocket} = 23 | gen_tcp:listen(11012, [binary, {packet, 4}, {reuseaddr, true}]), 24 | test1_api_accpet(ListeningSocket). 25 | 26 | test1_api_accpet(ListeningSocket) -> 27 | {ok, ApiSocket} = gen_tcp:accept(ListeningSocket), 28 | Pid = spawn(?MODULE, test1_api_proc, [ApiSocket]), 29 | gen_tcp:controlling_process(ApiSocket, Pid), 30 | test1_api_accpet(ListeningSocket). 31 | 32 | test1_api_proc(ApiSocket) -> 33 | receive 34 | {tcp, ApiSocket, Bin} -> 35 | test1_api_send(ApiSocket, binary_to_term(Bin)), 36 | test1_api_proc(ApiSocket) 37 | end. 38 | 39 | test1_api_send(ApiSocket, {list, 0 = _Bucket}) -> 40 | Data4 = #data{ 41 | key = ("item-4"), 42 | bucket = 0, 43 | last_modified = now(), 44 | checksum = erlang:md5(<<"item-4">>) 45 | }, 46 | gen_tcp:send(ApiSocket, term_to_binary({list_of_data, [Data4]})); 47 | test1_api_send(ApiSocket, {list, 3 = _Bucket}) -> 48 | Data1 = #data{ 49 | key = ("item-1"), 50 | bucket = 3, 51 | last_modified = now(), 52 | checksum = erlang:md5(<<"item-1">>) 53 | }, 54 | Data3 = #data{ 55 | key = ("item-3"), 56 | bucket = 3, 57 | last_modified = now(), 58 | checksum = erlang:md5(<<"item-3">>) 59 | }, 60 | gen_tcp:send(ApiSocket, term_to_binary({list_of_data, [Data1, Data3]})); 61 | test1_api_send(ApiSocket, {get, #data{key="item-3", bucket=3}}) -> 62 | Data3 = #data{ 63 | key = "item-3", 64 | bucket = 3, 65 | last_modified = now(), 66 | checksum = erlang:md5(<<"value-3">>), 67 | flags = "0", 68 | value = (<<"value-3">>) 69 | }, 70 | gen_tcp:send(ApiSocket, term_to_binary(Data3)); 71 | test1_api_send(ApiSocket, {get, #data{key="item-4", bucket=0}}) -> 72 | Data4 = #data{ 73 | key = "item-4", 74 | bucket = 0, 75 | last_modified = now(), 76 | checksum = erlang:md5(<<"value-4">>), 77 | flags = "0", 78 | value = (<<"value-4">>)}, 79 | gen_tcp:send(ApiSocket, term_to_binary(Data4)). 80 | 81 | test1() -> []. 82 | test1(_Conf) -> 83 | kai_config:start_link([ 84 | {hostname, "localhost"}, 85 | {rpc_port, 11011}, 86 | {n, 3}, 87 | {number_of_buckets, 8}, 88 | {number_of_virtual_nodes, 2}, 89 | {store, ets} 90 | ]), 91 | kai_hash:start_link(), 92 | kai_store:start_link(), 93 | kai_connection:start_link(), 94 | kai_sync:start_link(), 95 | 96 | {replaced_buckets, _ReplacedBuckets} = 97 | kai_hash:update_nodes([{?NODE2, ?INFO}], []), 98 | 99 | Data1 = #data{ 100 | key = ("item-1"), 101 | bucket = 3, 102 | last_modified = now(), 103 | checksum = erlang:md5(<<"value-1">>), 104 | flags = "0", 105 | value = (<<"value-1">>) 106 | }, 107 | kai_store:put(Data1), 108 | 109 | spawn_link(?MODULE, test1_api_start, []), 110 | 111 | kai_sync:update_bucket(3), 112 | 113 | timer:sleep(100), 114 | 115 | {list_of_data, ListOfData} = kai_store:list(3), 116 | ?assertEqual(2, length(ListOfData)), 117 | ?assert(lists:keymember("item-3", 2, ListOfData)), 118 | 119 | kai_sync:delete_bucket(3), 120 | 121 | timer:sleep(100), 122 | 123 | {list_of_data, ListOfData2} = kai_store:list(3), 124 | ?assertEqual(0, length(ListOfData2)), 125 | 126 | timer:sleep(1100), % timeout and update bucket-0 by kai_hash:choose_bucket_randomly/0 127 | 128 | {list_of_data, ListOfData3} = kai_store:list(0), 129 | ?assertEqual(1, length(ListOfData3)), 130 | ?assert(lists:keymember("item-4", 2, ListOfData3)), 131 | 132 | kai_sync:stop(), 133 | kai_connection:stop(), 134 | kai_store:stop(), 135 | kai_hash:stop(), 136 | kai_config:stop(). 137 | -------------------------------------------------------------------------------- /test/kai_tcp_server_SUITE.erl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -module(kai_tcp_server_SUITE). 14 | -compile(export_all). 15 | -export([init/1, handle_call/3]). % for echo server 16 | 17 | -include("ct.hrl"). 18 | -include("kai.hrl"). 19 | -include("kai_test.hrl"). 20 | 21 | sequences() -> 22 | [{sequences1, [testcase1, testcase2, testcase3, testcase4]}]. 23 | 24 | all() -> 25 | [{sequence, sequences1}]. 26 | 27 | init_per_testcase(testcase4, Config) -> 28 | start_server(10), 29 | Config; 30 | 31 | init_per_testcase(_TestCase, Config) -> 32 | start_server(1), 33 | Config. 34 | 35 | start_server(MaxProcesses) -> 36 | kai_tcp_server:start_link( 37 | ?MODULE, [], #tcp_server_option{max_processes=MaxProcesses} 38 | ). 39 | 40 | end_per_testcase(_TestCase, _Config) -> 41 | kai_tcp_server:stop(), 42 | ok. 43 | 44 | testcase1() -> []. 45 | testcase1(_Conf) -> 46 | normal_test(), 47 | ok. 48 | 49 | normal_test() -> 50 | {ok, Socket} = connect_to_echo_server(), 51 | gen_tcp:send(Socket, <<"hello\r\n">>), 52 | case gen_tcp:recv(Socket, 0) of 53 | {ok, <<"hello\r\n">>} -> ok; 54 | _HelloError -> ct:fail(bad_echo_value) 55 | end, 56 | gen_tcp:send(Socket, <<"bye\r\n">>), 57 | case gen_tcp:recv(Socket, 0) of 58 | {ok, <<"cya\r\n">>} -> ok; 59 | _ByeError -> ct:fail(bad_return_value) 60 | end, 61 | gen_tcp:close(Socket), 62 | ok. 63 | 64 | testcase2() -> []. 65 | testcase2(_Conf) -> 66 | {ok, Socket} = connect_to_echo_server(), 67 | gen_tcp:send(Socket, <<"error\r\n">>), 68 | {error, closed} = gen_tcp:recv(Socket, 0), 69 | gen_tcp:close(Socket), 70 | normal_test(), % check the echo server rebooted. 71 | ok. 72 | 73 | testcase3() -> []. 74 | testcase3(_Conf) -> 75 | lists:foreach(fun (_N) -> 76 | {ok, Socket} = connect_to_echo_server(), 77 | gen_tcp:close(Socket) 78 | end, lists:seq(1, 10000)), 79 | ok. 80 | 81 | testcase4() -> []. 82 | testcase4(_Conf) -> 83 | Sockets = lists:map(fun (_N) -> 84 | {ok, Socket} = connect_to_echo_server(), 85 | Socket 86 | end, lists:seq(1, 5)), 87 | timer:sleep(100), % wait for increment. 88 | case kai_tcp_server:info(curr_connections) of 89 | 5 -> ok; 90 | Error -> 91 | ct:comment(io:format("bad_info:~p", [Error])), 92 | ct:fail(bad_info) 93 | end, 94 | lists:foreach(fun (Socket) -> gen_tcp:close(Socket) end, Sockets), 95 | ok. 96 | 97 | connect_to_echo_server() -> 98 | gen_tcp:connect( 99 | {127,0,0,1}, 11211, [binary, {packet, line}, {active, false}] 100 | ). 101 | 102 | % echo server 103 | init(_Args) -> {ok, {}}. 104 | 105 | handle_call(_Socket, <<"bye\r\n">>, State) -> 106 | {close, <<"cya\r\n">>, State}; 107 | handle_call(_Socket, <<"error\r\n">>, State) -> 108 | (fun(X) -> 1 / X end)(0), % to throw a bad arithmetic exception 109 | {close, <<"error\r\n">>, State}; 110 | handle_call(_Socket, Data, State) -> 111 | {reply, Data, State}. 112 | 113 | -------------------------------------------------------------------------------- /test/kai_version_SUITE.erl: -------------------------------------------------------------------------------- 1 | %% Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | %% use this file except in compliance with the License. You may obtain a copy of 3 | %% the License at 4 | %% 5 | %% http://www.apache.org/licenses/LICENSE-2.0 6 | %% 7 | %% Unless required by applicable law or agreed to in writing, software 8 | %% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | %% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | %% License for the specific language governing permissions and limitations under 11 | %% the License. 12 | 13 | -module(kai_version_SUITE). 14 | -compile(export_all). 15 | 16 | -include("kai.hrl"). 17 | -include("kai_test.hrl"). 18 | 19 | init_per_testcase(_TestCase, Conf) -> 20 | kai_config:start_link([ 21 | {hostname, "localhost"}, 22 | {rpc_port, 11011}, 23 | {memcache_port, 11211}, 24 | {max_connections, 2}, 25 | {n, 1}, {r, 1}, {w, 1}, 26 | {number_of_buckets, 8}, 27 | {number_of_virtual_nodes, 2} 28 | ]), 29 | kai_version:start_link(), 30 | Conf. 31 | 32 | end_per_testcase(_TestCase, _Conf) -> 33 | kai_config:stop(), 34 | kai_version:stop(), 35 | ok. 36 | 37 | all() -> [test_update, test_order, 38 | test_cas_unique1, 39 | test_cas_unique2, 40 | test_cas_unique7, 41 | test_cas_unique16 42 | ]. 43 | 44 | test_update() -> []. 45 | test_update(_Conf) -> 46 | VClock1 = vclock:increment(kai_config:get(node), vclock:fresh()), 47 | 48 | Data1 = #data{ 49 | vector_clocks = VClock1 50 | }, 51 | 52 | {ok, Data2} = kai_version:update(Data1), 53 | ct:log(test_update, "Data2.vector_clocks: ~p~n", [Data2#data.vector_clocks]), 54 | ?assert(is_list(Data2#data.vector_clocks)), 55 | ?assert(vclock:descends(Data2#data.vector_clocks, Data1#data.vector_clocks)), 56 | ?assertNot(vclock:descends(Data1#data.vector_clocks, Data2#data.vector_clocks)), 57 | 58 | ok. 59 | 60 | test_order() -> []. 61 | test_order(_Conf) -> 62 | VClock1 = vclock:increment(node1, vclock:fresh()), 63 | Data1 = #data{ 64 | vector_clocks = VClock1 65 | }, 66 | 67 | %% trivial case 68 | ?assertEqual(1, length(kai_version:order([Data1]))), 69 | 70 | %% two concurrent data 71 | Data2 = Data1#data{ 72 | vector_clocks = vclock:increment(otherNode, vclock:fresh()) 73 | }, 74 | ListOfData12 = kai_version:order([Data1, Data2]), 75 | ?assertEqual(2, length(ListOfData12)), 76 | 77 | %% one data is dropped 78 | Data3 = Data1#data{ 79 | vector_clocks = vclock:increment(otherNode2, Data1#data.vector_clocks) 80 | }, 81 | ListOfData23 = kai_version:order([Data1, Data2, Data3]), 82 | ?assertEqual(2, length(ListOfData23)), 83 | ok. 84 | 85 | test_cas_unique1() -> []. 86 | test_cas_unique1(_Conf) -> 87 | Data1 = #data{ 88 | checksum = list_to_binary(all_bit_on(16)) 89 | }, 90 | {ok, CasUnique} = kai_version:cas_unique([Data1]), 91 | Expected = list_to_binary([<<1:4, 2#1111:4>>, all_bit_on(7)]), 92 | ?assertEqual(Expected, CasUnique), 93 | ok. 94 | 95 | test_cas_unique2() -> []. 96 | test_cas_unique2(_Conf) -> 97 | Data1 = #data{ 98 | checksum = <<16#FFFFFFFFFFFFFFFF:64, 0:64>> 99 | }, 100 | Data2 = #data{ 101 | checksum = <<0:64, 16#FFFFFFFFFFFFFFFF:64>> 102 | }, 103 | {ok, CasUnique} = kai_version:cas_unique([Data1, Data2]), 104 | Expected = <<2:4, 2#1111:4, 16#FFFFFFF:26, 0:30>>, 105 | ?assertEqual(Expected, CasUnique), 106 | ok. 107 | 108 | test_cas_unique7() -> []. 109 | test_cas_unique7(_Conf) -> 110 | %% trunc(60/7) = 8 111 | ListOfData = lists:map(fun (I) -> 112 | #data{checksum = <>} 113 | end, 114 | lists:seq(1,7)), 115 | {ok, CasUnique} = kai_version:cas_unique(ListOfData), 116 | Expected = <<7:4, 1:8, 2:8, 3:8, 4:8, 5:8, 6:8, 7:8, 0:4>>, 117 | ?assertEqual(Expected, CasUnique), 118 | ok. 119 | 120 | test_cas_unique16() -> []. 121 | test_cas_unique16(_Conf) -> 122 | %% 16 exceeds 4bit range (2#1111 = 15) 123 | ListOfData = lists:map(fun (I) -> 124 | #data{checksum = <>} 125 | end, 126 | lists:seq(1,16)), 127 | {error, Reason} = kai_version:cas_unique(ListOfData), 128 | ?assert(string:str(Reason, "16") > 0), 129 | ok. 130 | 131 | all_bit_on(Bytes) -> 132 | lists:duplicate(Bytes, 16#FF). 133 | 134 | all_bit_off(Bytes) -> 135 | lists:duplicate(Bytes, 0). 136 | -------------------------------------------------------------------------------- /test/vclock_SUITE.erl: -------------------------------------------------------------------------------- 1 | %% Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | %% use this file except in compliance with the License. You may obtain a copy of 3 | %% the License at 4 | %% 5 | %% http://www.apache.org/licenses/LICENSE-2.0 6 | %% 7 | %% Unless required by applicable law or agreed to in writing, software 8 | %% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | %% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | %% License for the specific language governing permissions and limitations under 11 | %% the License. 12 | 13 | -module(vclock_SUITE). 14 | -compile(export_all). 15 | 16 | -include("kai.hrl"). 17 | -include("kai_test.hrl"). 18 | 19 | all() -> [test_desceds_in_single_node]. 20 | 21 | test_desceds_in_single_node() -> []. 22 | 23 | %% @doc Serves as both a trivial test and some example code. 24 | test_desceds_in_single_node(_Conf) -> 25 | A1 = vclock:fresh(), 26 | A2 = vclock:increment(a, A1), 27 | ?assert(vclock:descends(A2,A1)), 28 | A3 = vclock:increment(a, A2), 29 | ?assert(vclock:descends(A3,A2)), 30 | ?assert(vclock:descends(A3,A1)), 31 | ok. 32 | 33 | test_concurrent() -> []. 34 | 35 | test_concurrent(_Conf) -> 36 | A1 = vclock:fresh(), 37 | B1 = vclock:fresh(), 38 | A2 = vclock:increment(a, A1), 39 | B2 = vclock:increment(b, B1), 40 | ?assertNot(vclock:descends(A2,B2)), 41 | ?assertNot(vclock:descends(B2,A2)), 42 | ok. 43 | 44 | test_merge() -> []. 45 | 46 | test_merge(_Conf) -> 47 | A1 = vclock:fresh(), 48 | B1 = vclock:fresh(), 49 | A2 = vclock:increment(a, A1), 50 | B2 = vclock:increment(b, B1), 51 | C1 = vclock:merge([A2, B2]), 52 | ?assert(vclock:descends(C1, A2)), 53 | ?assert(vclock:descends(C1, B2)), 54 | C2 = vclock:increment(c, C1), 55 | ?assertNot(vclock:descends(C2, A2)), 56 | ?assertNot(vclock:descends(C2, B2)), 57 | ?assertNot(vclock:descends(C2, C1)), 58 | ok. 59 | --------------------------------------------------------------------------------