├── .github └── workflows │ └── erlang.yml ├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── doc ├── repl.plu ├── repl.png └── repl_old.plu ├── eqc_int └── riak_repl2_rt_eqc.erl ├── include ├── riak_core_cluster.hrl ├── riak_core_connection.hrl ├── riak_repl.hrl ├── riak_repl_aae_fullsync.hrl └── rt_source_eqc.hrl ├── priv ├── .empty_for_hg └── riak_repl.schema ├── rebar.config ├── rebar3 ├── src ├── bounded_queue.erl ├── gen_leader.erl ├── riak_core_cluster_conn.erl ├── riak_core_cluster_conn_sup.erl ├── riak_core_cluster_mgr.erl ├── riak_core_cluster_mgr_sup.erl ├── riak_core_cluster_serv.erl ├── riak_core_connection.erl ├── riak_core_connection_mgr.erl ├── riak_core_connection_mgr_stats.erl ├── riak_core_service_conn.erl ├── riak_core_service_mgr.erl ├── riak_repl.app.src ├── riak_repl.erl ├── riak_repl.proto ├── riak_repl2_fs_node_reserver.erl ├── riak_repl2_fscoordinator.erl ├── riak_repl2_fscoordinator_serv.erl ├── riak_repl2_fscoordinator_serv_sup.erl ├── riak_repl2_fscoordinator_sup.erl ├── riak_repl2_fssink.erl ├── riak_repl2_fssink_pool.erl ├── riak_repl2_fssink_sup.erl ├── riak_repl2_fssource.erl ├── riak_repl2_fssource_sup.erl ├── riak_repl2_ip.erl ├── riak_repl2_leader.erl ├── riak_repl2_pg.erl ├── riak_repl2_pg_block_provider.erl ├── riak_repl2_pg_block_provider_sup.erl ├── riak_repl2_pg_block_requester.erl ├── riak_repl2_pg_block_requester_sup.erl ├── riak_repl2_pg_proxy.erl ├── riak_repl2_pg_proxy_sup.erl ├── riak_repl2_pg_sup.erl ├── riak_repl2_rt.erl ├── riak_repl2_rt_sup.erl ├── riak_repl2_rtframe.erl ├── riak_repl2_rtq.erl ├── riak_repl2_rtq_overload_counter.erl ├── riak_repl2_rtq_proxy.erl ├── riak_repl2_rtsink_conn.erl ├── riak_repl2_rtsink_conn_sup.erl ├── riak_repl2_rtsink_helper.erl ├── riak_repl2_rtsink_sup.erl ├── riak_repl2_rtsource_conn.erl ├── riak_repl2_rtsource_conn_sup.erl ├── riak_repl2_rtsource_helper.erl ├── riak_repl2_rtsource_sup.erl ├── riak_repl_aae_sink.erl ├── riak_repl_aae_source.erl ├── riak_repl_app.erl ├── riak_repl_bq.erl ├── riak_repl_bucket_type_util.erl ├── riak_repl_cinfo.erl ├── riak_repl_client_sup.erl ├── riak_repl_console.erl ├── riak_repl_cs.erl ├── riak_repl_fsm_common.erl ├── riak_repl_fullsync_helper.erl ├── riak_repl_fullsync_worker.erl ├── riak_repl_keylist_client.erl ├── riak_repl_keylist_server.erl ├── riak_repl_leader.erl ├── riak_repl_leader_helper.erl ├── riak_repl_listener_sup.erl ├── riak_repl_migration.erl ├── riak_repl_pb_get.erl ├── riak_repl_ring.erl ├── riak_repl_ring_handler.erl ├── riak_repl_rtenqueue.erl ├── riak_repl_server_sup.erl ├── riak_repl_stats.erl ├── riak_repl_sup.erl ├── riak_repl_tcp_client.erl ├── riak_repl_tcp_server.erl ├── riak_repl_util.erl ├── riak_repl_web.erl ├── riak_repl_wm_rtenqueue.erl └── riak_repl_wm_stats.erl └── test ├── fixup_test.erl ├── riak_core_cluster_conn_eqc.erl ├── riak_core_cluster_mgr_sup_tests.erl ├── riak_core_cluster_mgr_tests.erl ├── riak_core_connection_mgr_tests.erl ├── riak_core_connection_tests.erl ├── riak_core_service_mgr_tests.erl ├── riak_repl2_rt_source_sink_tests.erl ├── riak_repl2_rtq_eqc.erl ├── riak_repl2_rtq_tests.erl ├── riak_repl_cs_eqc.erl ├── riak_repl_schema_tests.erl ├── riak_repl_sup_tests.erl ├── riak_repl_test_util.erl ├── rt_sink_eqc.erl └── rt_source_helpers.erl /.github/workflows/erlang.yml: -------------------------------------------------------------------------------- 1 | name: Erlang CI 2 | 3 | on: 4 | push: 5 | branches: [ develop ] 6 | pull_request: 7 | branches: [ develop ] 8 | 9 | 10 | jobs: 11 | 12 | build: 13 | 14 | name: Test on ${{ matrix.os }} with OTP ${{ matrix.otp }} 15 | runs-on: ${{ matrix.os }} 16 | 17 | strategy: 18 | fail-fast: false 19 | matrix: 20 | otp: [22, 24, 25] 21 | os: [ubuntu-latest] 22 | # OTP lower than 23 does not run on ubuntu-latest (22.04), see 23 | # https://github.com/erlef/setup-beam#compatibility-between-operating-system-and-erlangotp 24 | exclude: 25 | - otp: 22 26 | os: ubuntu-latest 27 | include: 28 | - otp: 22 29 | os: ubuntu-20.04 30 | 31 | steps: 32 | - uses: lukka/get-cmake@latest 33 | - uses: actions/checkout@v2 34 | - name: Install dependencies (Ubuntu) 35 | if: ${{ startsWith(matrix.os, 'ubuntu') }} 36 | run: | 37 | sudo apt-get -qq update 38 | sudo apt-get -qq install libsnappy-dev libc6-dev 39 | - name: Install Erlang/OTP 40 | uses: erlef/setup-beam@v1 41 | with: 42 | otp-version: ${{ matrix.otp }} 43 | - name: Compile 44 | run: ./rebar3 compile 45 | - name: Run xref and dialyzer 46 | run: ./rebar3 do xref, dialyzer 47 | - name: Run eunit 48 | run: ./rebar3 as gha do eunit 49 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ebin/*.beam 2 | ebin/*.app 3 | deps 4 | .eunit 5 | .eqc/ 6 | .eqc-info 7 | .qc/ 8 | current_counterexample.eqc 9 | rebar3.crashdump 10 | erl_crash.dump 11 | repl_leader_eqc_data/ 12 | .local_dialyzer_plt 13 | tags 14 | dialyzer_unhandled_warnings 15 | dialyzer_warnings 16 | .rebar/ 17 | _build/ 18 | rebar.lock 19 | .DS_Store 20 | include/riak_repl_pb.hrl 21 | src/riak_repl_pb.erl 22 | undefined 23 | data/* 24 | .rebar3/erlcinfo 25 | test/*.beam 26 | log/*.log 27 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: compile rel cover test dialyzer eqc 2 | REBAR=./rebar3 3 | 4 | compile: 5 | $(REBAR) compile 6 | 7 | clean: 8 | rm -f include/*_pb.hrl 9 | rm -f src/*_pb.erl 10 | $(REBAR) clean 11 | 12 | cover: 13 | $(REBAR) eunit --cover 14 | $(REBAR) cover 15 | 16 | test: compile 17 | $(REBAR) eunit 18 | 19 | dialyzer: 20 | $(REBAR) dialyzer 21 | 22 | xref: 23 | $(REBAR) xref 24 | 25 | eqc: 26 | $(REBAR) as test eqc --testing_budget 120 27 | $(REBAR) as eqc eunit 28 | 29 | check: test dialyzer xref 30 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | riak_repl 2 | ========= 3 | 4 | Riak MDC Replication 5 | 6 | --- 7 | # Pull Request template 8 | 9 | ### Testing 10 | - [ ] manual verification of code 11 | - [ ] eunit (w/ gist of output) 12 | - [ ] EQC (w/ gist of output) 13 | - [ ] riak_test (w/ gist of output) 14 | - [ ] Dialyzer 15 | - [ ] XRef 16 | - [ ] Coverage reports 17 | 18 | ### Documentation 19 | - [ ] internal docs (design docs) 20 | - [ ] external docs (docs.basho.com) 21 | - [ ] man pages 22 | 23 | 24 | --- 25 | 26 | # New Feature Deliverables 27 | 28 | - design documentation + diagrams 29 | - nothing formal 30 | - to help out during support, "this is how xyz works" 31 | - eunit tests 32 | - riak_tests 33 | - EQC + Pulse tests 34 | - tests at scale and under heavy load 35 | - Boston Cluster or AWS 36 | - notes for public documentation 37 | - for the docs team 38 | - 39 | 40 | --- 41 | 42 | # BEAM release process 43 | 44 | 1. git tag the specific commit(s) that will be released 45 | 2. run all eunit tests, EQC tests, store the output in a gist. 46 | 3. if possible, run all riak_tests for replication 47 | 4. record specific commit(s) that the beam targets in a README.txt file 48 | 5. create a tar file. 49 | - Note that OSX will include a hidden directory in the tar file. Find the knob to prevent those files from being added to the .tar file, or build/test the beams on Linux. (you can use 'find' to pipe the exact files you want into the tar, see: https://github.com/basho/node_package/blob/develop/priv/templates/fbsd/Makefile#L27 for an example of using -rf with a pipe) 50 | - include the README.txt file from the step above 51 | 6. once .tar is built, calc an MD5 to pass along with the file 52 | 7. create an entry on https://github.com/basho/internal_wiki/wiki/Releases page 53 | - include: 54 | - link to the gist output 55 | - version of Erlang the beams were built with 56 | - MD5 of the file 57 | - link to compiled beams 58 | 8. notify client services + TAMs 59 | 9. port the PR to the `develop` branch if applicable 60 | 61 | 62 | -------------------------------------------------------------------------------- /doc/repl.plu: -------------------------------------------------------------------------------- 1 | Replication Message Sequence Chart(s) 2 | C.Tilt 3 | 26-July-2012 4 | 5 | Scope 6 | ----- 7 | There are three protocols now. The original (Andy Gross) is now called syncV1. 8 | Next, there was a refactor for Riak 1.1 (Andrew Thompson) that pulled apart the real-time and full-sync protocols into separate files and made it somewhat pluggable. Recently, there was an optimization to use Bloom filters in the fold operation (Andrew T.) to allow the GETs to proceed in disk-order to aliviate disk 9 | seeking overhead. [Note: this was a symptom of sorting the hashed key/value list 10 | to enable a difference generator; but, the sorted list was no longer in disk 11 | order and caused wild head seek thrash on spinning media.] 12 | 13 | The message sequences shown here are examples of the Bloom-filtered protocol, 14 | which as of this writing has not shipped yet (not scheduled for 1.2, but more 15 | likely a point release after being vetted by AT&T and possibly others). 16 | 17 | There are notes in the header of riak_repl_keylist_server.erl about 18 | replication's strategy. This document is an attempt to diagram it's design and 19 | implemenation at the message level. 20 | 21 | Terminology 22 | ---------- 23 | s,server := the "listener" side. listens for connections from client "sites" 24 | c,client := the "site" side. makes a connection to the server. 25 | 26 | [] := identifies a concurrent process 27 | * := process is created 28 | X := process exited 29 | () := state the process is currently in. 30 | (-) := the process remains in the same state ("-" means current). 31 | 32 | Full-sync 33 | -------- 34 | Either client or server can initiate the full-sync. If something on the server 35 | does, it sends a "start" to the client to tell it to begin full-sync. So either 36 | way, the show starts when the clients engages. For this MSC, we show what 37 | happens once the client starts full-sync. 38 | 39 | 40 | @startuml repl.png 41 | box "Listener on\nprimary cluster" 42 | participant "s:key-lister" 43 | participant server 44 | end box 45 | box "Site on\nbackup cluster" #LightBlue 46 | participant client 47 | participant "c:key-lister" 48 | end box 49 | 50 | note over server: (wait_for_partition) 51 | note over client: (wait_for_fullsync) 52 | 53 | [->server : start 54 | note over server #FFAAAA: Do all nodes\nsupport Bloom? 55 | alt YES 56 | 57 | server -> client : start_fullsync 58 | note over client #FFAAAA: P..PP = build partition list 59 | client -> client : continue 60 | 61 | note over client: (request_partition) 62 | client -> server: {partition,P} 63 | client -> "c:key-lister": spawn(make_keylist) 64 | activate "c:key-lister" 65 | "c:key-lister" -> client: {ok,Reg} 66 | note over "c:key-lister" #FFAAAA 67 | build Keylist 68 | sort Keylist 69 | N = number of keys 70 | end note 71 | "c:key-lister"->client: {keylist_built,N} 72 | destroy "c:key-lister" 73 | 74 | server -> "s:key-lister": spawn(make_keylist) 75 | activate "s:key-lister" 76 | "s:key-lister" -> server: {ok,Reg} 77 | note over server: (build_keylist) 78 | note over "s:key-lister" #FFAAAA 79 | build Keylist 80 | sort Keylist 81 | N = number of keys 82 | end note 83 | "s:key-lister" -> server: {keylist_built,N} 84 | destroy "s:key-lister" 85 | 86 | server -> client: {kl_exchange, P} 87 | note over server: (wait_keylist) 88 | 89 | client -> client: continue 90 | 91 | note over client: (send_keylist) 92 | loop N % 1000 93 | loop 1000 times 94 | client -> server: {kl_hunk, KeyData} 95 | client -> client: continue 96 | end loop 97 | client -> server: kl_wait 98 | ...wait for ack from server... 99 | server -> client: kl_ack 100 | client -> client: continue 101 | end loop 102 | client -> server: kl_eof 103 | note over client: (wait_ack) 104 | 105 | 'server spawns the differencer helper 106 | server -> "s:key-lister": spawn(diff_stream) 107 | activate "s:key-lister" 108 | "s:key-lister" -> server: {ok,Reg} 109 | note over "server" #FFAAAA: NumDiffs > KEY_LIST_THRESHOLD 110 | note over server: (diff_bloom) 111 | loop keys in Keylist 112 | "s:key-lister" -> server: {merkle_diff,{K,B}} 113 | note over server #FFAAAA: ebloom:insert({K,B}) 114 | "s:key-lister" -> server: diff_paused 115 | server -> "s:key-lister": diff_resume 116 | end loop 117 | "s:key-lister" -> server: diff_done 118 | destroy "s:key-lister" 119 | 120 | 'server start Bloom fold 121 | server -> "s:key-lister": spawn(bloom_fold,P) 122 | activate "s:key-lister" 123 | loop for key in all keys of partition P 124 | loop for diff_batch_size keys 125 | alt Bloom contains key 126 | "s:key-lister" -> server : {diff_obj, Obj} 127 | server -> client : {fs_diff_obj, Obj} 128 | 129 | client -> "c:key-lister": "do_put(Obj)" 130 | activate "c:key-lister" 131 | destroy "c:key-lister" 132 | else 133 | end 134 | end loop 135 | "s:key-lister" -> server: bloom_paused 136 | 137 | ' implement latency reduction with multiple windows in flight 138 | note over "server" #FFAAAA: did server receive\nprevious diff_ack? 139 | alt YES 140 | server -> client: diff_ack 141 | server -> "s:key-lister": bloom_resume 142 | else NO 143 | note over "s:key-lister" #FFAAAA: wait for resume 144 | client -> server: diff_ack 145 | server -> client: diff_ack 146 | server -> "s:key-lister": bloom_resume 147 | end 148 | end loop 149 | "s:key-lister" -> server: diff_exchanged 150 | destroy "s:key-lister" 151 | 152 | server -> client: diff_done 153 | note over server: "(wait_for_partition)" 154 | 155 | 'This partition is finished! Do the next partition 156 | note over client: "(request_partition)" 157 | client -> client: continue 158 | else NO 159 | note over server: (wait_for_partition) 160 | end 161 | 162 | @enduml -------------------------------------------------------------------------------- /doc/repl.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/basho/riak_repl/53841f400cfba6fcde9760c94c72bb10dbe8b84e/doc/repl.png -------------------------------------------------------------------------------- /doc/repl_old.plu: -------------------------------------------------------------------------------- 1 | Replication Message Sequence Chart(s) 2 | C.Tilt 3 | 26-July-2012 4 | 5 | Scope 6 | ----- 7 | There are three protocols now. The original (Andy Gross) is now called syncV1. 8 | Next, there was a refactor for Riak 1.1 (Andrew Thompson) that pulled apart the real-time and full-sync protocols into separate files and made it somewhat pluggable. Recently, there was an optimization to use Bloom filters in the fold operation (Andrew T.) to allow the GETs to proceed in disk-order to aliviate disk 9 | seeking overhead. [Note: this was a symptom of sorting the hashed key/value list 10 | to enable a difference generator; but, the sorted list was no longer in disk 11 | order and caused wild head seek thrash on spinning media.] 12 | 13 | The message sequences shown here are examples of the Bloom-filtered protocol, 14 | which as of this writing has not shipped yet (not scheduled for 1.2, but more 15 | likely a point release after being vetted by AT&T and possibly others). 16 | 17 | There are notes in the header of riak_repl_keylist_server.erl about 18 | replication's strategy. This document is an attempt to diagram it's design and 19 | implemenation at the message level. 20 | 21 | Terminology 22 | ---------- 23 | s,server := the "listener" side. listens for connections from client "sites" 24 | c,client := the "site" side. makes a connection to the server. 25 | 26 | [] := identifies a concurrent process 27 | * := process is created 28 | X := process exited 29 | () := state the process is currently in. 30 | (-) := the process remains in the same state ("-" means current). 31 | 32 | Full-sync 33 | -------- 34 | Either client or server can initiate the full-sync. If something on the server 35 | does, it sends a "start" to the client to tell it to begin full-sync. So either 36 | way, the show starts when the clients engages. For this MSC, we show what 37 | happens once the client starts full-sync. 38 | 39 | 40 | @startuml repl_old.png 41 | box "Listener on\nprimary cluster" 42 | participant "s:key-lister" 43 | participant server 44 | end box 45 | box "Site on\nbackup cluster" #LightBlue 46 | participant client 47 | participant "c:key-lister" 48 | end box 49 | 50 | note over server: (wait_for_partition) 51 | note over client: (wait_for_fullsync) 52 | 53 | [->server : start 54 | note over server #FFAAAA: Do all nodes\nsupport Bloom? 55 | alt YES 56 | 57 | server -> client : start_fullsync 58 | note over client #FFAAAA: P..PP = build partition list 59 | client -> client : continue 60 | 61 | note over client: (request_partition) 62 | client -> server: {partition,P} 63 | client -> "c:key-lister": spawn(make_keylist) 64 | activate "c:key-lister" 65 | "c:key-lister" -> client: {ok,Reg} 66 | note over "c:key-lister" #FFAAAA 67 | build Keylist 68 | sort Keylist 69 | N = number of keys 70 | end note 71 | "c:key-lister"->client: {keylist_built,N} 72 | destroy "c:key-lister" 73 | 74 | server -> "s:key-lister": spawn(make_keylist) 75 | activate "s:key-lister" 76 | "s:key-lister" -> server: {ok,Reg} 77 | note over server: (build_keylist) 78 | note over "s:key-lister" #FFAAAA 79 | build Keylist 80 | sort Keylist 81 | N = number of keys 82 | end note 83 | "s:key-lister" -> server: {keylist_built,N} 84 | destroy "s:key-lister" 85 | 86 | server -> client: {kl_exchange, P} 87 | note over server: (wait_keylist) 88 | 89 | client -> client: continue 90 | 91 | note over client: (send_keylist) 92 | loop N % 1000 93 | loop 1000 times 94 | client -> server: {kl_hunk, KeyData} 95 | client -> client: continue 96 | end loop 97 | client -> server: kl_wait 98 | ...wait for ack from server... 99 | server -> client: kl_ack 100 | client -> client: continue 101 | end loop 102 | client -> server: kl_eof 103 | note over client: (wait_ack) 104 | 105 | 'server spawns the differencer helper 106 | server -> "s:key-lister": spawn(diff_stream) 107 | activate "s:key-lister" 108 | "s:key-lister" -> server: {ok,Reg} 109 | note over server: (diff_keylist) 110 | loop keys in Keylist 111 | loop for diff_batch_size keys 112 | "s:key-lister" -> server: {merkle_diff,{K,B}} 113 | server -> client : {fs_diff_obj, Obj} 114 | end loop 115 | "s:key-lister" -> server: diff_paused 116 | 117 | note over server #FFAAAA: did server receive\nprevious diff_ack? 118 | alt YES 119 | server -> client: diff_ack 120 | server -> "s:key-lister": diff_resume 121 | else NO 122 | note over "s:key-lister" #FFAAAA: wait for resume 123 | client -> server: diff_ack 124 | server -> client: diff_ack 125 | server -> "s:key-lister": diff_resume 126 | end 127 | 128 | end loop 129 | "s:key-lister" -> server: diff_done 130 | destroy "s:key-lister" 131 | 132 | server -> client: diff_done 133 | note over server: "(wait_for_partition)" 134 | 135 | 'This partition is finished! Do the next partition 136 | note over client: "(request_partition)" 137 | client -> client: continue 138 | else NO 139 | note over server: (-) 140 | end 141 | 142 | @enduml -------------------------------------------------------------------------------- /include/riak_core_cluster.hrl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Riak Core Cluster Manager 4 | %% 5 | %% Copyright (c) 2013 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | -define(CLUSTER_MANAGER_SERVER, riak_core_cluster_manager). 23 | -define(CLUSTER_MGR_SERVICE_ADDR, {"0.0.0.0", 9085}). 24 | -define(CM_CALL_TIMEOUT, 2000). 25 | -define(CLUSTER_NAME_LOCATOR_TYPE, cluster_by_name). 26 | -define(CLUSTER_ADDR_LOCATOR_TYPE, cluster_by_addr). 27 | 28 | -define(CLUSTER_PROTO_ID, cluster_mgr). 29 | -type(clustername() :: string()). 30 | -------------------------------------------------------------------------------- /include/riak_core_connection.hrl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Riak Core Connection Manager 4 | %% 5 | %% Copyright (c) 2013 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | %% handshake messages to safely initiate a connection. Let's not accept 24 | %% a connection to a telnet session by accident! 25 | -define(CTRL_REV, {1,0}). 26 | -define(CTRL_HELLO, <<"riak-ctrl:hello">>). 27 | -define(CTRL_TELL_IP_ADDR, <<"riak-ctrl:ip_addr">>). 28 | -define(CTRL_ACK, <<"riak-ctrl:ack">>). 29 | -define(CTRL_ASK_NAME, <<"riak-ctrl:ask_name">>). 30 | -define(CTRL_ASK_MEMBERS, <<"riak-ctrl:ask_members">>). 31 | -define(CTRL_ALL_MEMBERS, <<"riak-ctrl:all_members">>). 32 | 33 | 34 | -define(CONNECTION_SETUP_TIMEOUT, 10000). 35 | -define(INITIAL_CONNECTION_RESPONSE_TIMEOUT, 15000). 36 | 37 | 38 | 39 | -define(CTRL_OPTIONS, [binary, 40 | {keepalive, true}, 41 | {nodelay, true}, 42 | {packet, 4}, 43 | {reuseaddr, true}, 44 | {active, false}]). 45 | 46 | %% Tcp options shared during the connection and negotiation phase 47 | -define(CONNECT_OPTIONS, [binary, 48 | {keepalive, true}, 49 | {nodelay, true}, 50 | {packet, 4}, 51 | {reuseaddr, true}, 52 | {active, false}]). 53 | 54 | -type(ip_addr_str() :: string()). 55 | -type(ip_portnum() :: non_neg_integer()). 56 | -type(ip_addr() :: {ip_addr_str(), ip_portnum()}). 57 | -type(tcp_options() :: [any()]). 58 | 59 | -type(proto_id() :: atom()). 60 | -type(rev() :: non_neg_integer()). %% major or minor revision number 61 | -type(proto() :: {proto_id(), {rev(), rev()}}). %% e.g. {myproto, 1, 0} 62 | -type(protoprefs() :: {proto_id(), [{rev(), rev()}]}). 63 | 64 | 65 | %% Function = fun(Socket, Transport, Protocol, Args) -> ok 66 | %% Protocol :: proto() 67 | -type(service_started_callback() :: fun((inet:socket(), module(), proto(), [any()]) -> no_return())). 68 | 69 | %% Host protocol spec 70 | -type(hostspec() :: {protoprefs(), {tcp_options(), module(), service_started_callback(), [any()]}}). 71 | 72 | %% Client protocol spec 73 | -type(clientspec() :: {protoprefs(), {tcp_options(), module(),[any()]}}). 74 | 75 | 76 | %% Scheduler strategies tell the connection manager how distribute the load. 77 | %% 78 | %% Client scheduler strategies 79 | %% --------------------------- 80 | %% default := service side decides how to choose node to connect to. limits number of 81 | %% accepted connections to max_nb_cons(). 82 | %% askme := the connection manager will call your client for a custom protocol strategy 83 | %% and likewise will expect that the service side has plugged the cluster 84 | %% manager with support for that custom strategy. UNSUPPORTED so far. 85 | %% TODO: add a behaviour for the client to implement that includes this callback. 86 | %% Service scheduler strategies 87 | %% ---------------------------- 88 | %% round_robin := choose the next available node on cluster to service request. limits 89 | %% the number of accepted connections to max_nb_cons(). 90 | %% custom := service must provide a strategy to the cluster manager for choosing nodes 91 | %% UNSUPPORTED so far. Should we use a behaviour for the service module? 92 | -type(max_nb_cons() :: non_neg_integer()). 93 | -type(client_scheduler_strategy() :: default | askme). 94 | -type(service_scheduler_strategy() :: {round_robin, max_nb_cons()} | custom). 95 | 96 | %% service manager statistics, can maybe get shared by other layers too 97 | -record(stats, {open_connections = 0 : non_negative_integer() 98 | }). 99 | 100 | 101 | -type(cluster_finder_fun() :: fun(() -> {ok,node()} | {error, term()})). 102 | -------------------------------------------------------------------------------- /include/riak_repl.hrl: -------------------------------------------------------------------------------- 1 | %% Riak EnterpriseDS 2 | %% Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved. 3 | -define(REPL_FSM_TIMEOUT, 15000). 4 | -define(REPL_QUEUE_TIMEOUT, 1000). 5 | -define(REPL_MERK_TIMEOUT, infinity). 6 | -define(REPL_CONN_RETRY, 30000). 7 | -define(DEFAULT_REPL_PORT, 9010). 8 | -define(NEVER_SYNCED, {0, 0, 0}). 9 | -define(MERKLE_BUFSZ, 1048576). 10 | -define(MERKLE_CHUNKSZ, 65536). 11 | -define(REPL_DEFAULT_QUEUE_SIZE, 104857600). 12 | -define(REPL_DEFAULT_MAX_PENDING, 5). 13 | -define(REPL_DEFAULT_ACK_FREQUENCY, 5). 14 | -define(FSM_SOCKOPTS, [{packet, 4}, {send_timeout, 300000}]). 15 | -define(REPL_VERSION, 3). 16 | -define(LEGACY_STRATEGY, keylist). 17 | -define(KEEPALIVE_TIME, 60000). 18 | -define(PEERINFO_TIMEOUT, 60000). 19 | -define(ELECTION_TIMEOUT, 60000). 20 | -define(TCP_MON_RT_APP, repl_rt). 21 | -define(TCP_MON_FULLSYNC_APP, repl_fullsync). 22 | -define(TCP_MON_PROXYGET_APP, proxy_get). 23 | -define(DEFAULT_REPL_MODE, mode_repl12). 24 | -define(DEFAULT_SOURCE_PER_NODE, 1). 25 | -define(DEFAULT_SOURCE_PER_CLUSTER, 5). 26 | -define(DEFAULT_MAX_SINKS_NODE, 1). 27 | %% How many times during a fullsync we should try a partition 28 | -define(DEFAULT_SOURCE_RETRIES, infinity). 29 | %% How many times we should retry when failing a reservation 30 | -define(DEFAULT_RESERVE_RETRIES, 0). 31 | %% How many times during a fullsync we should retry a partion that has sent 32 | %% a 'soft_exit' message to the coordinator 33 | -define(DEFAULT_SOURCE_SOFT_RETRIES, 100). 34 | %% how long must elaspse before a partition fullsync is retried after 35 | %% a soft exit 36 | -define(DEFAULT_SOURCE_RETRY_WAIT_SECS, 60). 37 | %% 20 seconds. sources should claim within 5 seconds, but give them a little more time 38 | -define(RESERVATION_TIMEOUT, (20 * 1000)). 39 | -define(DEFAULT_MAX_FS_BUSIES_TOLERATED, 10). 40 | -define(RETRY_WHEREIS_INTERVAL, 1000). 41 | -define(CONSOLE_RPC_TIMEOUT, 5000). 42 | -define(RETRY_AAE_LOCKED_INTERVAL, 1000). 43 | -define(DEFAULT_FULLSYNC_STRATEGY, keylist). %% keylist | aae 44 | %% the following are keys for bucket types related meta data 45 | -define(BT_META_TYPED_BUCKET, typed_bucket). 46 | -define(BT_META_TYPE, bucket_type). 47 | -define(BT_META_PROPS_HASH, properties_hash_val). 48 | 49 | -type(ip_addr_str() :: string()). 50 | -type(ip_portnum() :: non_neg_integer()). 51 | -type(repl_addr() :: {ip_addr_str(), ip_portnum()}). 52 | -type(repl_addrlist() :: [repl_addr()]). 53 | -type(repl_socket() :: port() | ssl:sslsocket()). 54 | -type(repl_sitename() :: string()). 55 | -type(repl_sitenames() :: [repl_sitename()]). 56 | -type(repl_ns_pair() :: {node(), repl_sitename()}). 57 | -type(repl_ns_pairs() :: [repl_ns_pair()]). 58 | -type(repl_np_pair() :: {repl_sitename(), pid()}). 59 | -type(repl_np_pairs() :: [repl_np_pair()]). 60 | -type(repl_node_sites() :: {node(), [{repl_sitename(), pid()}]}). 61 | -type(ring() :: tuple()). 62 | 63 | -ifdef(namespaced_types). 64 | -type riak_repl_dict() :: dict:dict(). 65 | -else. 66 | -type riak_repl_dict() :: dict(). 67 | -endif. 68 | 69 | -type(repl_config() :: riak_repl_dict()|undefined). 70 | %% wire_version() is an atom that defines which wire format a binary 71 | %% encoding of one of more riak objects is packaged into. For details of 72 | %% the to_wire() and from_wire() operations, see riak_repl_util.erl. 73 | %% Also see analagous binary_version() in riak_object, which is carried 74 | %% inside the wire format in "BinObj". w0 implies v0. w1 imples v1. 75 | %% w2 still uses v1, but supports encoding the type information 76 | %% for bucket types. 77 | -type(wire_version() :: w0 %% simple term_to_binary() legacy encoding 78 | | w1 %% <>. 81 | | w2). %% <>. 85 | 86 | -record(peer_info, { 87 | riak_version :: string(), %% version number of the riak_kv app 88 | repl_version :: string(), %% version number of the riak_kv app 89 | ring :: ring() %% instance of riak_core_ring() 90 | }). 91 | 92 | -record(fsm_state, { 93 | socket :: repl_socket(), %% peer socket 94 | sitename :: repl_sitename(), %% peer sitename 95 | my_pi :: #peer_info{}, %% local peer_info 96 | client :: tuple(), %% riak local_client 97 | partitions = [] :: list(), %% list of local partitions 98 | work_dir :: string() %% working directory 99 | }). 100 | 101 | -record(repl_listener, { 102 | nodename :: atom(), %% cluster-local node name 103 | listen_addr :: repl_addr() %% ip/port to bind/listen on 104 | }). 105 | 106 | -record(repl_site, { 107 | name :: repl_sitename(), %% site name 108 | addrs=[] :: repl_addrlist(),%% list of ip/ports to connect to 109 | last_sync=?NEVER_SYNCED :: tuple() 110 | }). 111 | 112 | -record(nat_listener, { 113 | nodename :: atom(), %% cluster-local node name 114 | listen_addr :: repl_addr(), %% ip/port to bind/listen on 115 | nat_addr :: repl_addr() %% ip/port that nat bind/listens to 116 | }). 117 | 118 | -define(REPL_HOOK_BNW, {struct, 119 | [{<<"mod">>, <<"riak_repl2_rt">>}, 120 | {<<"fun">>, <<"postcommit">>}]}). 121 | 122 | -define(REPL_HOOK12, {struct, 123 | [{<<"mod">>, <<"riak_repl_leader">>}, 124 | {<<"fun">>, <<"postcommit">>}]}). 125 | 126 | -define(REPL_MODES, [{mode_repl12,?REPL_HOOK12}, {mode_repl13,?REPL_HOOK_BNW}]). 127 | 128 | 129 | -define(LONG_TIMEOUT, 120*1000). 130 | 131 | -define(V2REPLDEP, "DEPRECATION NOTICE: The replication protocol you are currently using in this cluster has been deprecated and will be unsupported and removed some time after the Riak Enterprise 2.1 release. Please upgrade to the latest replication protocol as soon as possible. If you need assistance migrating contact Basho Client Services or follow the instructions in our documentation ( http://docs.basho.com/riakee/latest/cookbooks/Multi-Data-Center-Replication-UpgradeV2toV3/ )."). 132 | -------------------------------------------------------------------------------- /include/riak_repl_aae_fullsync.hrl: -------------------------------------------------------------------------------- 1 | -define(MSG_INIT, 1). 2 | -define(MSG_LOCK_TREE, 2). 3 | -define(MSG_UPDATE_TREE, 3). 4 | -define(MSG_GET_AAE_BUCKET, 4). 5 | -define(MSG_GET_AAE_SEGMENT, 5). 6 | -define(MSG_REPLY, 6). 7 | -define(MSG_PUT_OBJ, 7). 8 | -define(MSG_GET_OBJ, 8). 9 | -define(MSG_COMPLETE, 9). 10 | -define(AAE_FULLSYNC_REPLY_TIMEOUT, 60000). %% 1 minute to hear from remote sink 11 | -------------------------------------------------------------------------------- /include/rt_source_eqc.hrl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% 4 | %% Copyright (c) 2014 Basho Technologies, Inc. All Rights Reserved. 5 | %% 6 | %% This file is provided to you under the Apache License, 7 | %% Version 2.0 (the "License"); you may not use this file 8 | %% except in compliance with the License. You may obtain 9 | %% a copy of the License at 10 | %% 11 | %% http://www.apache.org/licenses/LICENSE-2.0 12 | %% 13 | %% Unless required by applicable law or agreed to in writing, 14 | %% software distributed under the License is distributed on an 15 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | %% KIND, either express or implied. See the License for the 17 | %% specific language governing permissions and limitations 18 | %% under the License. 19 | %% 20 | %% ------------------------------------------------------------------- 21 | 22 | %% all_remotes defines the list of remote clients the test will manage 23 | 24 | %% -define(all_remotes, ["a", "b", "c", "d", "e"]). 25 | -define(all_remotes, ["a"]). 26 | 27 | -record(state, { 28 | remotes_available = ?all_remotes, 29 | seq = 0, 30 | master_queue = [], 31 | sources = [] % [#src_state{}] 32 | }). 33 | 34 | -record(src_state, { 35 | pids, % {SourcePid, SinkPid} 36 | version, 37 | skips = 0, 38 | offset = 0, 39 | unacked_objects = [] 40 | }). 41 | -------------------------------------------------------------------------------- /priv/.empty_for_hg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/basho/riak_repl/53841f400cfba6fcde9760c94c72bb10dbe8b84e/priv/.empty_for_hg -------------------------------------------------------------------------------- /priv/riak_repl.schema: -------------------------------------------------------------------------------- 1 | %%-*- mode: erlang -*- 2 | %% Replication config 3 | 4 | %% @doc Path (relative or absolute) to the working directory for the 5 | %% replication process 6 | {mapping, "mdc.data_root", "riak_repl.data_root", [ 7 | {default, "{{repl_data_root}}"} 8 | ]}. 9 | 10 | %% @doc The cluster manager will listen for connections from remote 11 | %% clusters on this ip and port. Every node runs one cluster manager, 12 | %% but only the cluster manager running on the cluster_leader will 13 | %% service requests. This can change as nodes enter and leave the 14 | %% cluster. The value is a combination of an IP address (**not 15 | %% hostname**) followed by a port number 16 | {mapping, "mdc.cluster_manager", "riak_core.cluster_mgr", [ 17 | {default, {"{{cluster_manager_ip}}", {{cluster_manager_port}} }}, 18 | {datatype, ip} 19 | ]}. 20 | 21 | %% @doc The hard limit of fullsync workers that will be running on the 22 | %% source side of a cluster across all nodes on that cluster for a 23 | %% fullsync to a sink cluster. This means if one has configured 24 | %% fullsync for two different clusters, both with a 25 | %% max_fssource_cluster of 5, 10 fullsync workers can be in 26 | %% progress. Only affects nodes on the source cluster on which this 27 | %% parameter is defined via the configuration file or command line 28 | {mapping, "mdc.max_fssource_cluster", "riak_repl.max_fssource_cluster", [ 29 | {datatype, integer}, 30 | {default, 5} 31 | ]}. 32 | 33 | %% @doc Limits the number of fullsync workers that will be running on 34 | %% each individual node in a source cluster. This is a hard limit for 35 | %% all fullsyncs enabled; additional fullsync configurations will not 36 | %% increase the number of fullsync workers allowed to run on any node. 37 | %% Only affects nodes on the source cluster on which this parameter is 38 | %% defined via the configuration file or command line 39 | {mapping, "mdc.max_fssource_node", "riak_repl.max_fssource_node", [ 40 | {datatype, integer}, 41 | {default, 1} 42 | ]}. 43 | 44 | %% @doc Limits the number of "soft_exist" that the fullsynce 45 | %% coordinator will handle before failing a partition from 46 | %% fullsync. The soft_retries is per-fullsync, not per-partition. 47 | %% Only affects nodes on the source cluster on which this parameter is 48 | %% defined via the configuration file 49 | {mapping, "mdc.max_fssource_soft_retries", "riak_repl.max_fssource_soft_retries", [ 50 | {datatype, integer}, 51 | {default, 100} 52 | ]}. 53 | 54 | %% @doc Adds a retry wait time. To be used in conjunction with 55 | %% soft_retries. When a partition fails to fullsync with a soft_exit, 56 | %% it is added to a queue to be retried. The retry wait time is the 57 | %% minimum amount of time to elapse before a fullsync is re-attempted 58 | %% on that partition. An example of usage: If the remote partition's 59 | %% AAE tree is being re-built it can take many minutes, even 60 | %% hours. There is no point in rapidly re-trying the same partition 61 | %% `max_fssource_soft_retries' times in rapid 62 | %% succession. fssource_retry_wait * max_fssource_soft_retries is the 63 | %% maximum amount of time that can pass before fullsync discards a 64 | %% partition. 65 | {mapping, "mdc.fssource_retry_wait", "riak_repl.fssource_retry_wait", [ 66 | {datatype, {duration, s}}, 67 | {default, "60s"} 68 | ]}. 69 | 70 | %% @doc Limits the number of fullsync workers allowed to run on each 71 | %% individual node in a sink cluster. This is a hard limit for all 72 | %% fullsync sources interacting with the sink cluster. Thus, multiple 73 | %% simultaneous source connections to the sink cluster will have to 74 | %% share the sink node's number of maximum connections. Only affects 75 | %% nodes on the sink cluster on which this parameter is defined via 76 | %% the configuration file or command line. 77 | {mapping, "mdc.max_fssink_node", "riak_repl.max_fssink_node", [ 78 | {datatype, integer}, 79 | {default, 1} 80 | ]}. 81 | 82 | %% @doc Whether to initiate a fullsync on initial connection from the 83 | %% secondary cluster 84 | {mapping, "mdc.fullsync_on_connect", "riak_repl.fullsync_on_connect", [ 85 | {datatype, {enum, [true, false]}}, 86 | {default, true} 87 | ]}. 88 | 89 | %% @doc a single integer value representing the duration to wait in 90 | %% minutes between fullsyncs, or a list of {clustername, 91 | %% time_in_minutes} pairs for each sink participating in fullsync 92 | %% replication. 93 | {mapping, "mdc.fullsync_interval.$cluster_name", "riak_repl.fullsync_interval", [ 94 | {datatype, {duration, ms}}, 95 | {include_default, "all"}, 96 | {commented, "30m"} 97 | ]}. 98 | 99 | {translation, 100 | "riak_repl.fullsync_interval", 101 | fun(Conf) -> 102 | Minute = fun(Millis) -> Millis div 60000 end, 103 | FullSyncIntervals = cuttlefish_variable:filter_by_prefix("mdc.fullsync_interval", Conf), 104 | case proplists:get_value(["mdc", "fullsync_interval", "all"], FullSyncIntervals) of 105 | undefined -> 106 | [ {list_to_atom(Name), Minute(Value)} || {["mdc", "fullsync_interval", Name], Value} <- FullSyncIntervals]; 107 | X -> Minute(X) 108 | end 109 | end}. 110 | 111 | %% @doc The maximum size the realtime replication queue can grow to 112 | %% before new objects are dropped. Defaults to 100MB. Dropped objects 113 | %% will need to be replication with a fullsync. 114 | {mapping, "mdc.rtq_max_bytes", "riak_repl.rtq_max_bytes", [ 115 | {datatype, bytesize}, 116 | {default, "100MB"} 117 | ]}. 118 | 119 | %% @doc Enable Riak CS proxy_get and block filter. 120 | {mapping, "mdc.proxy_get", "riak_repl.proxy_get", [ 121 | {datatype, {enum, [on, off]}}, 122 | {default, off} 123 | ]}. 124 | 125 | {translation, 126 | "riak_repl.proxy_get", 127 | fun(Conf) -> 128 | case cuttlefish:conf_get("mdc.proxy_get", Conf) of 129 | on -> enabled; 130 | off -> disabled; 131 | _ -> disabled 132 | end 133 | end}. 134 | 135 | %% @doc A heartbeat message is sent from the source to the sink every 136 | %% heartbeat_interval. Setting heartbeat_interval to undefined 137 | %% disables the realtime heartbeat. This feature is only available in 138 | %% Riak Enterprise 1.3.2+. 139 | {mapping, "mdc.realtime.heartbeat_interval", "riak_repl.rt_heartbeat_interval", [ 140 | {datatype, {duration, s}}, 141 | {default, "15s"} 142 | ]}. 143 | 144 | %% @doc If a heartbeat response is not received in 145 | %% rt_heartbeat_timeout seconds, then the source connection exits and 146 | %% will be re-established. This feature is only available in Riak 147 | %% Enterprise 1.3.2+. 148 | {mapping, "mdc.realtime.heartbeat_timeout", "riak_repl.rt_heartbeat_timeout", [ 149 | {datatype, {duration, s}}, 150 | {default, "15s"} 151 | ]}. 152 | 153 | %% @doc By default, fullsync replication will try to coordinate with other 154 | %% riak subsystems that may be contending for the same resources. This will help 155 | %% to prevent system response degradation under times of heavy load from multiple 156 | %% background tasks. To disable background coordination, set this parameter to false. 157 | %% Enterprise 2.0+. 158 | {mapping, "mdc.fullsync.use_bg_manager", "riak_repl.fullsync_use_background_manager", [ 159 | {datatype, {enum, [true, false]}}, 160 | {default, false}, 161 | hidden 162 | ]}. 163 | 164 | %% @doc How frequently the stats for fullsync source processes should be 165 | %% gathered. Requests for fullsync status always returned the most recently 166 | %% gathered data, and thus can be at most as old as this value. 167 | {mapping, "mdc.fullsync.stat_refresh_interval", "riak_repl.fullsync_stat_refresh_interval", [ 168 | {datatype, {duration, ms}}, 169 | {commented, "1m"} 170 | ]}. 171 | 172 | -------------------------------------------------------------------------------- /rebar.config: -------------------------------------------------------------------------------- 1 | %%-*- mode: erlang -*- 2 | 3 | {cover_enabled, true}. 4 | 5 | {erl_opts, [debug_info, 6 | warnings_as_errors, 7 | {i, "./_build/default/plugins/gpb/include"}, 8 | {platform_define, "^[0-9]+", namespaced_types}, 9 | {platform_define, "^[2-9][1-9](.?[0-9]*)", otp21}, 10 | {platform_define, "^[2-9][2-9](.?[0-9]*)", otp22}, 11 | {platform_define, "^[2-9][5-9](.?[0-9]*)", otp25}]}. 12 | 13 | {erl_first_files, ["src/gen_leader.erl"]}. 14 | 15 | {xref_checks,[ 16 | undefined_function_calls, 17 | undefined_functions]}. 18 | 19 | {xref_queries, [{"(XC - UC) || (XU - X - B - cluster_info : Mod)", []}]}. 20 | 21 | {eunit_opts, [verbose]}. 22 | 23 | {deps, [ 24 | {ranch, {git, "https://github.com/ninenines/ranch.git", {tag, "1.8.0"}}}, 25 | {ebloom,{git, "https://github.com/basho/ebloom.git", {tag, "2.1.0"}}}, 26 | {riak_kv, {git, "https://github.com/basho/riak_kv.git", {branch, "develop"}}} 27 | ]}. 28 | 29 | {plugins, [{rebar3_gpb_plugin, {git, "https://github.com/basho/rebar3_gpb_plugin", {tag, "2.15.1+riak.3.0.4"}}}, 30 | {eqc_rebar, {git, "https://github.com/Quviq/eqc-rebar", {branch, "master"}}}]}. 31 | 32 | {gpb_opts, [{module_name_suffix, "_pb"}, 33 | {msg_name_to_lower, true}, 34 | {i, "src"}]}. 35 | 36 | {provider_hooks, [{pre, [{compile, {protobuf, compile}}]}]}. 37 | 38 | {dialyzer, [{plt_apps, all_deps}]}. 39 | 40 | {profiles, 41 | [{test, [{deps, [meck]}]}, 42 | {eqc, [{deps, [meck]}, {erl_opts, [{d, 'EQC'}]}]}, 43 | {gha, [{erl_opts, [{d, 'GITHUBEXCLUDE'}]}]} 44 | ]}. 45 | 46 | {edoc_opts, [{preprocess, true}]}. 47 | -------------------------------------------------------------------------------- /rebar3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/basho/riak_repl/53841f400cfba6fcde9760c94c72bb10dbe8b84e/rebar3 -------------------------------------------------------------------------------- /src/bounded_queue.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% bounded_queue: a size-bounded FIFO 4 | %% 5 | %% Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | %% @doc A simple size-bounded FIFO queue. 23 | 24 | -module(bounded_queue). 25 | -author('Andy Gross '). 26 | 27 | -export([new/1, 28 | in/2, 29 | out/1, 30 | byte_size/1, 31 | max_size/1, 32 | len/1, 33 | dropped_count/1]). 34 | 35 | -ifdef(TEST). 36 | -include_lib("eunit/include/eunit.hrl"). 37 | -endif. 38 | 39 | -ifdef(namespaced_types). 40 | -type bounded_queue_queue() :: queue:queue(). 41 | -else. 42 | -type bounded_queue_queue() :: queue(). 43 | -endif. 44 | 45 | -record(bq, { 46 | m=0 :: non_neg_integer(), %% maximum size of queue, in bytes. 47 | s=0 :: non_neg_integer(), %% current size of queue, in bytes. 48 | q=queue:new() :: bounded_queue_queue(), %% underlying queue. 49 | d=0 :: non_neg_integer()} %% dropped item count 50 | ). 51 | 52 | -type(bounded_queue() :: #bq{}). 53 | -type(bounded_queue_element() :: [binary(), ...] | binary()). 54 | 55 | %% @doc Create a new queue with maximum size in bytes of MaxSize. 56 | -spec new(non_neg_integer()) -> bounded_queue(). 57 | new(MaxSize) when is_integer(MaxSize) -> #bq{m=MaxSize, s=0, q=queue:new()}. 58 | 59 | %% @doc Add an item to the queue. 60 | -spec in(bounded_queue(), bounded_queue_element()) -> bounded_queue(). 61 | in(BQ=#bq{m=Max, q=Q, d=D}, Item) when is_binary(Item) -> 62 | ItemSize = element_size(Item), 63 | case ItemSize > Max of 64 | true -> 65 | DroppedObjs = queue:to_list(Q), 66 | _ = [ riak_repl_util:dropped_realtime_hook(DroppedObj) || 67 | DroppedObj <- DroppedObjs ], 68 | BQ#bq{q=queue:from_list([Item]),s=ItemSize,d=queue:len(Q)+D}; 69 | false -> 70 | make_fit(BQ, Item, ItemSize) 71 | end; 72 | in(BQ=#bq{m=Max, q=Q, d=D}, [H|_T] = Items) when is_binary(H) -> 73 | ItemSize = element_size(Items), 74 | case ItemSize > Max of 75 | true -> 76 | DroppedObjs = queue:to_list(Q), 77 | _ = [ riak_repl_util:dropped_realtime_hook(DroppedObj) 78 | || DroppedObj <- DroppedObjs ], 79 | BQ#bq{q=queue:from_list([Items]),s=ItemSize,d=queue:len(Q)+D}; 80 | false -> 81 | make_fit(BQ, Items, ItemSize) 82 | end. 83 | 84 | %% @doc Remove an item from the queue. 85 | -spec out(bounded_queue()) -> {{'value',bounded_queue_element()}|'empty', bounded_queue()}. 86 | out(BQ=#bq{q=Q,s=Size}) -> 87 | case queue:out(Q) of 88 | {empty, _} -> {empty, BQ}; 89 | {{value, Item}, NewQ} -> 90 | {{value, Item}, BQ#bq{s=Size-element_size(Item),q=NewQ}} 91 | end. 92 | 93 | %% @spec byte_size(bounded_queue()) -> non_neg_integer() 94 | %% @doc The size of the queue, in bytes. 95 | -spec byte_size(bounded_queue()) -> non_neg_integer(). 96 | byte_size(#bq{s=Size}) -> Size. 97 | 98 | max_size(#bq{m=Max}) -> Max. 99 | 100 | %% @spec len(bounded_queue()) -> non_neg_integer() 101 | %% @doc The number of items in the queue. 102 | -spec len(bounded_queue()) -> non_neg_integer(). 103 | len(#bq{q=Q}) -> queue:len(Q). 104 | 105 | %% @spec dropped_count(bounded_queue()) -> non_neg_integer() 106 | %% @doc The number of items dropped from the queue due to the size bound. 107 | -spec dropped_count(bounded_queue()) -> non_neg_integer(). 108 | dropped_count(#bq{d=D}) -> D. 109 | 110 | 111 | make_fit(BQ=#bq{s=Size,m=Max,d=D}, Item, ItemSize) when (ItemSize+Size>Max) -> 112 | {DroppedItem, NewQ} = out(BQ), 113 | riak_repl_util:dropped_realtime_hook(DroppedItem), 114 | make_fit(NewQ#bq{d=D+1}, Item, ItemSize); 115 | make_fit(BQ=#bq{q=Q, s=Size}, Item, ItemSize) -> 116 | BQ#bq{q=queue:in(Item, Q), s=Size+ItemSize}. 117 | 118 | element_size([H|_T] = Element) when is_binary(H) -> 119 | lists:foldl(fun(E, Acc) -> 120 | Acc + erlang:byte_size(E) 121 | end, 0, Element); 122 | element_size(Element) when is_binary(Element) -> 123 | erlang:byte_size(Element). 124 | 125 | 126 | 127 | %% =================================================================== 128 | %% EUnit tests 129 | %% =================================================================== 130 | -ifdef(TEST). 131 | 132 | initialization_test() -> 133 | Q = new(16), 134 | 0 = bounded_queue:len(Q), 135 | 0 = bounded_queue:byte_size(Q), 136 | 0 = bounded_queue:dropped_count(Q), 137 | Q. 138 | 139 | in_test() -> 140 | Q0 = initialization_test(), 141 | B = <<1:128/integer>>, 142 | Q1 = in(Q0, B), 143 | 1 = bounded_queue:len(Q1), 144 | 16 = bounded_queue:byte_size(Q1), 145 | 0 = bounded_queue:dropped_count(Q1), 146 | Q1. 147 | 148 | out_test() -> 149 | Q0 = in_test(), 150 | {{value, <<1:128/integer>>}, Q1} = out(Q0), 151 | {empty, Q2} = out(Q1), 152 | 0 = bounded_queue:len(Q2), 153 | 0 = bounded_queue:byte_size(Q2), 154 | 0 = bounded_queue:dropped_count(Q2), 155 | Q2. 156 | 157 | maxsize_test() -> 158 | Q0 = out_test(), 159 | Q1 = in(Q0, <<1:64/integer>>), 160 | Q2 = in(Q1, <<2:64/integer>>), 161 | Q3 = in(Q2, <<3:64/integer>>), 162 | {{value, Item}, Q4} = out(Q3), 163 | <<2:64/integer>> = Item, 164 | 8 = bounded_queue:byte_size(Q4), 165 | 1 = bounded_queue:len(Q4), 166 | 1 = bounded_queue:dropped_count(Q4), 167 | Q4. 168 | 169 | largeitem_test() -> 170 | Q0 = initialization_test(), 171 | Q1 = in(Q0, <<1:256/integer>>), 172 | 32 = bounded_queue:byte_size(Q1), 173 | 1 = bounded_queue:len(Q1), 174 | 0 = bounded_queue:dropped_count(Q1), 175 | Q1. 176 | 177 | list_of_items_test() -> 178 | Q0 = initialization_test(), 179 | Q1 = in(Q0, [<<1:64/integer>>, <<2:64/integer>>]), 180 | 1 = bounded_queue:len(Q1), 181 | 16 = bounded_queue:byte_size(Q1), 182 | Q2 = in(Q1, [<<3:64/integer>>, <<4:64/integer>>]), 183 | 1 = bounded_queue:dropped_count(Q2), 184 | {{value, [<<3:64/integer>>, <<4:64/integer>>]}, Q3} = out(Q2), 185 | Q4 = in(Q3, [<<5:64/integer>>, <<6:64/integer>>, <<7:64/integer>>]), 186 | 24 = bounded_queue:byte_size(Q4), 187 | 1 = bounded_queue:len(Q4), 188 | Q5 = in(Q4, <<8:64/integer>>), 189 | 8 = bounded_queue:byte_size(Q5), 190 | 1 = bounded_queue:len(Q5), 191 | Q5. 192 | 193 | -endif. 194 | -------------------------------------------------------------------------------- /src/riak_core_cluster_conn_sup.erl: -------------------------------------------------------------------------------- 1 | %% Riak Core Cluster Manager Connection Supervisor 2 | %% Copyright 2007-2012 Basho Technologies, Inc. All Rights Reserved. 3 | %% 4 | %% Mission: ensure connections from a cluster manager to other clusters 5 | %% in order to resolve ip addresses into known clusters or to refresh 6 | %% the list of remote cluster members and observe their status. 7 | 8 | -module(riak_core_cluster_conn_sup). 9 | -behaviour(supervisor). 10 | 11 | -ifdef(TEST). 12 | -include_lib("eunit/include/eunit.hrl"). 13 | -endif. 14 | 15 | -include_lib("kernel/include/logger.hrl"). 16 | 17 | -export([start_link/0, 18 | add_remote_connection/1, remove_remote_connection/1, 19 | connections/0, is_connected/1 20 | ]). 21 | -export([init/1]). 22 | 23 | -define(SHUTDOWN, 5000). % how long to give cluster_conn processes to shutdown 24 | 25 | start_link() -> 26 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 27 | 28 | %% a remote connection if we don't already have one 29 | add_remote_connection(Remote) -> 30 | case is_connected(Remote) of 31 | false -> 32 | ?LOG_INFO("Connecting to remote cluster: ~p", [Remote]), 33 | ChildSpec = make_remote(Remote), 34 | supervisor:start_child(?MODULE, ChildSpec); 35 | _ -> 36 | ?LOG_INFO("Already connected to remote cluster: ~p", [Remote]), 37 | ok 38 | end. 39 | 40 | remove_remote_connection(Remote) -> 41 | ?LOG_DEBUG("Disconnecting from remote cluster at: ~p", [Remote]), 42 | %% remove supervised cluster connection 43 | _ = supervisor:terminate_child(?MODULE, Remote), 44 | _ = supervisor:delete_child(?MODULE, Remote), 45 | %% This seems hacky, but someone has to tell the connection manager to stop 46 | %% trying to reach this target if it hasn't connected yet. It's the supervised 47 | %% cluster connection that requests the connection, but it's going to die, so 48 | %% it can't un-connect itself. 49 | riak_core_connection_mgr:disconnect(Remote). 50 | 51 | connections() -> 52 | [{Remote, Pid} || {Remote, Pid, _, _} <- supervisor:which_children(?MODULE), is_pid(Pid)]. 53 | 54 | is_connected(Remote) -> 55 | Connections = connections(), 56 | lists:any(fun({R,_Pid}) -> R == Remote end, Connections). 57 | %not ([] == lists:filter(fun({R,_Pid}) -> R == Remote end, connections())). 58 | 59 | %% @private 60 | init([]) -> 61 | %% %% TODO: remote list of test addresses. 62 | %% Remotes = initial clusters or ip addrs from ring 63 | %% Children = [make_remote(Remote) || Remote <- Remotes], 64 | Children = [], 65 | {ok, {{one_for_one, 10, 10}, Children}}. 66 | 67 | make_remote(Remote) -> 68 | {Remote, {riak_core_cluster_conn, start_link, [Remote]}, 69 | transient, ?SHUTDOWN, worker, [riak_core_cluster_conn]}. 70 | -------------------------------------------------------------------------------- /src/riak_core_cluster_mgr_sup.erl: -------------------------------------------------------------------------------- 1 | %% Riak Core Cluster Manager Supervisor 2 | %% Copyright 2007-2012 Basho Technologies, Inc. All Rights Reserved. 3 | %% 4 | -module(riak_core_cluster_mgr_sup). 5 | -behaviour(supervisor). 6 | 7 | %% External exports 8 | -export([start_link/0]). 9 | 10 | %% supervisor callbacks 11 | -export([init/1]). 12 | 13 | %% @spec start_link() -> ServerRet 14 | %% @doc API for starting the supervisor. 15 | start_link() -> 16 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 17 | 18 | %% @spec init([]) -> SupervisorTree 19 | %% @doc supervisor callback. 20 | init([]) -> 21 | ClusterMgrDefaults = [fun riak_repl_app:cluster_mgr_member_fun/1, 22 | fun riak_repl_app:cluster_mgr_all_member_fun/1, 23 | fun riak_repl_app:cluster_mgr_write_cluster_members_to_ring/2, 24 | fun riak_repl_app:cluster_mgr_read_cluster_targets_from_ring/0], 25 | Processes = 26 | [%% TODO: move these to riak core 27 | %% Service Manager (inbound connections) 28 | {riak_core_service_mgr, {riak_core_service_mgr, start_link, []}, 29 | permanent, 5000, worker, [riak_core_service_mgr]}, 30 | %% Connection Manager Stats - don't start it from here. It gets registered 31 | %% {riak_core_connection_mgr_stats, {riak_core_connection_mgr_stats, start_link, []}, 32 | %% permanent, 5000, worker, [riak_core_connection_mgr_stats]}, 33 | %% Connection Manager (outbound connections) 34 | {riak_core_connection_mgr, {riak_core_connection_mgr, start_link, []}, 35 | permanent, 5000, worker, [riak_core_connection_mgr]}, 36 | %% Cluster Client Connection Supervisor 37 | {riak_core_cluster_conn_sup, {riak_core_cluster_conn_sup, start_link, []}, 38 | permanent, infinity, supervisor, [riak_core_cluster_conn_sup]}, 39 | %% Cluster Manager 40 | {riak_repl_cluster_mgr, {riak_core_cluster_mgr, start_link, ClusterMgrDefaults}, 41 | permanent, 5000, worker, [riak_core_cluster_mgr]}, 42 | {riak_core_tcp_mon, {riak_core_tcp_mon, start_link, []}, 43 | permanent, 5000, worker, [riak_core_tcp_mon]} 44 | ], 45 | {ok, {{rest_for_one, 9, 10}, Processes}}. 46 | -------------------------------------------------------------------------------- /src/riak_core_cluster_serv.erl: -------------------------------------------------------------------------------- 1 | -module(riak_core_cluster_serv). 2 | -behavior(gen_server). 3 | 4 | -include("riak_core_cluster.hrl"). 5 | -include("riak_core_connection.hrl"). 6 | 7 | -include_lib("kernel/include/logger.hrl"). 8 | 9 | -record(state, { 10 | transport, transport_msgs, socket, local_ver, remote_ver, remote_addr, remote_name 11 | }). 12 | 13 | % external api 14 | -export([ start_link/5 ]). 15 | % gen_server 16 | -export([ 17 | init/1, 18 | handle_call/3, handle_cast/2, handle_info/2, 19 | terminate/2, code_change/3 20 | ]). 21 | 22 | %% ========== 23 | %% public api 24 | %% ========== 25 | 26 | start_link(_Socket, _Transport, {error, _} = Error, _Args, _Props) -> 27 | Error; 28 | 29 | start_link(Socket, Transport, NegotiatedVers, _Args, Props) -> 30 | {ok, ClientAddr} = Transport:peername(Socket), 31 | RemoteClusterName = proplists:get_value(clustername, Props), 32 | ?LOG_DEBUG("Cluster Manager: accepted connection from cluster at ~p named ~p"), 33 | case gen_server:start_link(?MODULE, {Socket, Transport, NegotiatedVers, ClientAddr, RemoteClusterName, Props}, []) of 34 | {ok, Pid} -> 35 | Transport:controlling_process(Socket, Pid), 36 | ok = gen_server:cast(Pid, control_given), 37 | {ok, Pid}; 38 | Else -> 39 | Else 40 | end. 41 | 42 | %% ========== 43 | %% init 44 | %% ========== 45 | 46 | init({Socket, Transport, NegotiatedVers, ClientAddr, RemoteClusterName, _Props}) -> 47 | {ok, {cluster_mgr, LocalVer, RemoteVer}} = NegotiatedVers, 48 | State = #state{socket = Socket, transport = Transport, remote_addr = ClientAddr, remote_name = RemoteClusterName, local_ver = LocalVer, remote_ver = RemoteVer}, 49 | {ok, State}. 50 | 51 | %% ========== 52 | %% handle_call 53 | %% ========== 54 | 55 | handle_call(_Req, _From, State) -> 56 | {reply, {error, invalid}, State}. 57 | 58 | %% ========== 59 | %% handle_cast 60 | %% ========== 61 | 62 | handle_cast(control_given, State) -> 63 | #state{transport = Transport, socket = Socket} = State, 64 | ok = Transport:setopts(Socket, [{active, once}]), 65 | TransportMsgs = Transport:messages(), 66 | {noreply, State#state{transport_msgs = TransportMsgs}}; 67 | 68 | handle_cast(_Req, State) -> 69 | {noreply, State}. 70 | 71 | %% ========== 72 | %% handle_info 73 | %% ========== 74 | 75 | handle_info({TransOk, Socket, Data}, State = #state{transport_msgs = {TransOk, _, _}, socket = Socket}) when is_binary(Data) -> 76 | %Termed = binary_to_term(Data), 77 | handle_socket_info(Data, State#state.transport, State#state.socket, State); 78 | 79 | handle_info({TransClosed, Socket}, State = #state{socket = Socket, transport_msgs = {_, TransClosed, _}}) -> 80 | {stop, normal, State}; 81 | 82 | handle_info({TransError, Socket, Error}, State = #state{socket = Socket, transport_msgs = {_, _, TransError}}) -> 83 | {stop, {error, {connection_error, Error}}, State}. 84 | 85 | %% ========== 86 | %% terminate 87 | %% ========== 88 | 89 | terminate(_Why, _State) -> 90 | ok. 91 | 92 | %% ========== 93 | %% code_change 94 | %% ========== 95 | 96 | code_change(_OldVsn, State, _Extra) -> 97 | {ok, State}. 98 | 99 | %% ========== 100 | %% Internal 101 | %% ========== 102 | 103 | handle_socket_info(?CTRL_ASK_NAME, Transport, Socket, State) -> 104 | LocalName = riak_core_connection:symbolic_clustername(), 105 | ok = Transport:send(Socket, term_to_binary(LocalName)), 106 | ok = Transport:setopts(Socket, [{active, once}]), 107 | {noreply, State}; 108 | 109 | handle_socket_info(?CTRL_ASK_MEMBERS, Transport, Socket, State) -> 110 | case read_ip_address(Socket, Transport, State#state.remote_addr) of 111 | {ok, RemoteConnectedToIp} -> 112 | Members = gen_server:call(?CLUSTER_MANAGER_SERVER, {get_my_members, RemoteConnectedToIp}, infinity), 113 | ok = Transport:send(Socket, term_to_binary(Members)), 114 | ok = Transport:setopts(Socket, [{active, once}]), 115 | {noreply, State}; 116 | Else -> 117 | {stop, Else, State} 118 | end; 119 | 120 | handle_socket_info(?CTRL_ALL_MEMBERS, Transport, Socket, State) -> 121 | case read_ip_address(Socket, Transport, State#state.remote_addr) of 122 | {ok, RemoteConnectedToIp} -> 123 | Members = gen_server:call(?CLUSTER_MANAGER_SERVER, {get_all_members, RemoteConnectedToIp}, infinity), 124 | ok = Transport:send(Socket, term_to_binary(Members)), 125 | ok = Transport:setopts(Socket, [{active, once}]), 126 | {noreply, State}; 127 | Else -> 128 | {stop, Else, State} 129 | end; 130 | 131 | handle_socket_info(OtherData, _Transport, _Socket, State) -> 132 | ?LOG_WARNING("Some other data from the socket: ~p", [OtherData]), 133 | {stop, {error, unrecognized_request}, State}. 134 | 135 | read_ip_address(Socket, Transport, Remote) -> 136 | case Transport:recv(Socket, 0, ?CONNECTION_SETUP_TIMEOUT) of 137 | {ok, BinAddr} -> 138 | try binary_to_term(BinAddr, [safe]) of 139 | {IPAddrStr, IPPort} when is_list(IPAddrStr), is_integer(IPPort) -> 140 | {ok, {IPAddrStr, IPPort}}; 141 | InvalidTerm -> 142 | ?LOG_ERROR("Cluster Manager: failed to receive ip addr from remote ~p: invalid term recieved: ~p", [Remote, InvalidTerm]), 143 | {error, bad_address_format} 144 | catch 145 | error:badarg -> 146 | ?LOG_ERROR("Cluster Manager: failed to receive ip addr from remote ~p: unsafe binary to decode was sent", [Remote]), 147 | {error, unsafe_data} 148 | end; 149 | {error, closed} -> 150 | {error, closed}; 151 | Error -> 152 | ?LOG_ERROR("Cluster Manager: failed to receive ip addr from remote ~p: ~p", 153 | [Remote, Error]), 154 | Error 155 | end. 156 | 157 | -------------------------------------------------------------------------------- /src/riak_repl.app.src: -------------------------------------------------------------------------------- 1 | % -*- mode: erlang -*- 2 | {application, 3 | riak_repl, 4 | [{description, "Enterprise replication for Riak"}, 5 | {id, "riak_repl"}, 6 | {vsn, git}, 7 | {applications, [kernel, 8 | stdlib, 9 | sasl, 10 | crypto, 11 | ssl, 12 | ebloom, 13 | riak_core, 14 | riak_kv, 15 | riak_api, 16 | ranch]}, 17 | {registered, [riak_repl_connector_sup, 18 | riak_repl_leader, 19 | riak_repl_stats, 20 | riak_rep_sup]}, 21 | {mod, {riak_repl_app, []}}, 22 | {env, [ 23 | %% milliseconds to wait after checking all listeners 24 | {client_retry_timeout, 30000}, 25 | %% milliseconds to wait for successfull connect 26 | {client_connect_timeout, 15000}, 27 | 28 | {fullsync_on_connect, true}, 29 | % minutes 30 | {fullsync_interval, 360}, 31 | {data_root, "data/riak_repl"}, 32 | {merkle_bufsize, 1048576}, 33 | %% bytes 34 | {server_max_pending, 5}, 35 | {client_ack_frequency, 5}, 36 | {queue_size, 104857600}, 37 | {fullsync_strategies, [keylist]}, 38 | {min_get_workers, 5}, 39 | {max_get_workers, 100}, 40 | {min_put_workers, 5}, 41 | {max_put_workers, 100}, 42 | %% whether to issue gets directly against the vnode 43 | {vnode_gets, true}, 44 | %% How many fullsync diff objects to send before needing an 45 | %% ACK from the client. Setting this too high will clog your 46 | %% TCP buffers. 47 | {diff_batch_size, 100}, 48 | %% Exometer bootstrap 49 | {exometer_folsom_monitor, 50 | [{riak_core_connection_mgr_stats, riak_core_exo_monitor}, 51 | {riak_repl_stats, riak_core_exo_monitor} 52 | ]} 53 | ]} 54 | ]}. 55 | -------------------------------------------------------------------------------- /src/riak_repl.erl: -------------------------------------------------------------------------------- 1 | %% Riak EnterpriseDS 2 | %% Copyright 2007-2009 Basho Technologies, Inc. All Rights Reserved. 3 | -module(riak_repl). 4 | -author('Andy Gross '). 5 | -include("riak_repl.hrl"). 6 | 7 | -include_lib("kernel/include/logger.hrl"). 8 | 9 | -export([start/0, stop/0]). 10 | -export([install_hook/0, uninstall_hook/0]). 11 | -export([fixup/2]). 12 | -export([conditional_hook/3]). 13 | 14 | -ifdef(TEST). 15 | -include_lib("eunit/include/eunit.hrl"). 16 | -endif. 17 | 18 | start() -> 19 | riak_core_util:start_app_deps(riak_repl), 20 | application:start(riak_repl). 21 | 22 | %% @spec stop() -> ok 23 | stop() -> 24 | application:stop(riak_repl). 25 | 26 | install_hook() -> 27 | riak_kv_hooks:add_conditional_postcommit({?MODULE, conditional_hook}), 28 | riak_core_bucket:append_bucket_defaults([{repl, true}]), 29 | ok. 30 | 31 | uninstall_hook() -> 32 | riak_kv_hooks:del_conditional_postcommit({?MODULE, conditional_hook}), 33 | %% Cannot remove bucket defaults, best we can do is disable 34 | riak_core_bucket:append_bucket_defaults([{repl, false}]), 35 | ok. 36 | 37 | conditional_hook(_BucketType, _Bucket, BucketProps) -> 38 | RTEnabled = app_helper:get_env(riak_repl, rtenabled, false), 39 | BucketEnabled = not lists:member({repl, false}, BucketProps), 40 | case RTEnabled and BucketEnabled of 41 | true -> 42 | riak_repl_util:get_hooks_for_modes(); 43 | false -> 44 | false 45 | end. 46 | 47 | fixup(_Bucket, BucketProps) -> 48 | CleanPostcommit = strip_postcommit(BucketProps), 49 | RTEnabled = app_helper:get_env(riak_repl, rtenabled, false), 50 | case proplists:get_value(repl, BucketProps) of 51 | Val when (Val==true orelse Val==realtime orelse Val==both), 52 | RTEnabled == true -> 53 | ?LOG_DEBUG("Hooks for repl modes = ~p", [riak_repl_util:get_hooks_for_modes()]), 54 | UpdPostcommit = CleanPostcommit ++ riak_repl_util:get_hooks_for_modes(), 55 | 56 | {ok, lists:keystore(postcommit, 1, BucketProps, 57 | {postcommit, UpdPostcommit})}; 58 | _ -> 59 | %% Update the bucket properties 60 | UpdBucketProps = lists:keystore(postcommit, 1, BucketProps, 61 | {postcommit, CleanPostcommit}), 62 | {ok, UpdBucketProps} 63 | end. 64 | 65 | %% Get the postcommit hook from the bucket and strip any 66 | %% existing repl hooks. 67 | strip_postcommit(BucketProps) -> 68 | %% Get the current postcommit hook 69 | case proplists:get_value(postcommit, BucketProps, []) of 70 | X when is_list(X) -> 71 | CurrentPostcommit=X; 72 | {struct, _}=X -> 73 | CurrentPostcommit=[X] 74 | end, 75 | %% Add repl hook - make sure there are not duplicate entries 76 | AllHooks = [Hook || {_Mode, Hook} <- ?REPL_MODES], 77 | lists:filter(fun(H) -> not lists:member(H, AllHooks) end, CurrentPostcommit). 78 | 79 | 80 | -ifdef(TEST). 81 | 82 | -define(MY_HOOK1, {struct, 83 | [{<<"mod">>, <<"mymod1">>}, 84 | {<<"fun">>, <<"myhook1">>}]}). 85 | -define(MY_HOOK2, {struct, 86 | [{<<"mod">>, <<"mymod2">>}, 87 | {<<"fun">>, <<"myhook2">>}]}). 88 | strip_postcommit_test_() -> 89 | [?_assertEqual( 90 | [], % remember, returns the stipped postcommit from bprops 91 | lists:sort(strip_postcommit([{blah, blah}, {postcommit, []}]))), 92 | ?_assertEqual( 93 | [?MY_HOOK1, ?MY_HOOK2], 94 | lists:sort(strip_postcommit([{postcommit, [?REPL_HOOK_BNW, 95 | ?MY_HOOK1, 96 | ?REPL_HOOK_BNW, 97 | ?MY_HOOK2, 98 | ?REPL_HOOK12]}, 99 | {blah, blah}])))]. 100 | 101 | 102 | 103 | -endif. 104 | 105 | -------------------------------------------------------------------------------- /src/riak_repl.proto: -------------------------------------------------------------------------------- 1 | 2 | // Get Request - retrieve bucket/key 3 | message RpbReplGetReq { 4 | required bytes bucket = 1; 5 | required bytes key = 2; 6 | required bytes cluster_id = 3; 7 | optional uint32 r = 4; 8 | optional uint32 pr = 5; 9 | optional bool basic_quorum = 6; 10 | optional bool notfound_ok = 7; 11 | optional bytes if_modified = 8; // fail if the supplied vclock does not match 12 | optional bool head = 9; // return everything but the value 13 | optional bool deletedvclock = 10; // return the tombstone's vclock, if applicable 14 | optional bool sloppy_quorum = 11; // Experimental, may change/disappear 15 | optional uint32 n_val = 12; // Experimental, may change/disappear 16 | } 17 | 18 | // Get Cluster Id request 19 | message RpbReplGetClusterIdReq { 20 | } 21 | 22 | // Get Cluster Id response 23 | message RpbReplGetClusterIdResp { 24 | required bytes cluster_id = 1; 25 | } 26 | -------------------------------------------------------------------------------- /src/riak_repl2_fs_node_reserver.erl: -------------------------------------------------------------------------------- 1 | %% @doc Hold reservations for new sink processes on the node. It takes into 2 | %% account the running sinks and how many reservations there are for that node. 3 | -module(riak_repl2_fs_node_reserver). 4 | -include("riak_repl.hrl"). 5 | 6 | -include_lib("kernel/include/logger.hrl"). 7 | 8 | -behaviour(gen_server). 9 | -define(SERVER, ?MODULE). 10 | 11 | -record(state, { 12 | reservations = [] 13 | }). 14 | 15 | %% ------------------------------------------------------------------ 16 | %% API Function Exports 17 | %% ------------------------------------------------------------------ 18 | 19 | -export([start_link/0]). 20 | -export([reserve/1, unreserve/1, claim_reservation/1]). 21 | 22 | %% ------------------------------------------------------------------ 23 | %% gen_server Function Exports 24 | %% ------------------------------------------------------------------ 25 | 26 | -export([init/1, handle_call/3, handle_cast/2, handle_info/2, 27 | terminate/2, code_change/3]). 28 | 29 | %% ------------------------------------------------------------------ 30 | %% API Function Definitions 31 | %% ------------------------------------------------------------------ 32 | 33 | %% @doc Start the reservation server on the local node, registering to the 34 | %% module name. 35 | -spec start_link() -> {'ok', pid()}. 36 | start_link() -> 37 | gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). 38 | 39 | %% @doc Find the node for given partition, and issue a reservation on that node. 40 | %% If the node is down or a reservation process is not running, `down' is 41 | %% returned. A reservation that is unclaimed for 20 seconds is released. 42 | -spec reserve(Partition :: any()) -> 'ok' | 'busy' | 'down'. 43 | reserve(Partition) -> 44 | Node = get_partition_node(Partition), 45 | % I don't want to crash the caller if the node is down 46 | %% TODO: add explicit timeout to this call 47 | %% TODO: add timeout handling to catch? 48 | try gen_server:call({?SERVER, Node}, {reserve, Partition}) of 49 | Out -> Out 50 | catch 51 | exit:{noproc, _} -> 52 | down; 53 | exit:{{nodedown, _}, _} -> 54 | down 55 | end. 56 | 57 | %% @doc Release a reservation for the given partition on the correct 58 | %% node. If the node is down or a reservation process is not running, 59 | %% `down' is returned. 60 | -spec unreserve(Partition :: any()) -> 'ok' | 'busy' | 'down'. 61 | unreserve(Partition) -> 62 | Node = get_partition_node(Partition), 63 | try gen_server:call({?SERVER, Node}, {unreserve, Partition}) of 64 | Out -> 65 | Out 66 | catch 67 | exit:{noproc, _} -> 68 | down; 69 | exit:{{nodedown, _}, _} -> 70 | down 71 | end. 72 | 73 | %% @doc Indicates a reservation has been converted to a running sink. Usually 74 | %% used by a sink. 75 | -spec claim_reservation(Partition :: any()) -> 'ok'. 76 | claim_reservation(Partition) -> 77 | Node = get_partition_node(Partition), 78 | gen_server:cast({?SERVER, Node}, {claim_reservation, Partition}). 79 | 80 | 81 | %% ------------------------------------------------------------------ 82 | %% gen_server Function Definitions 83 | %% ------------------------------------------------------------------ 84 | 85 | %% @hidden 86 | init(_Args) -> 87 | {ok, #state{}}. 88 | 89 | %% @hidden 90 | handle_call({reserve, Partition}, _From, State) -> 91 | Kids = supervisor:which_children(riak_repl2_fssink_sup), 92 | Max = app_helper:get_env(riak_repl, max_fssink_node, ?DEFAULT_MAX_SINKS_NODE), 93 | Running = length(Kids), 94 | Reserved = length(State#state.reservations), 95 | if 96 | (Running + Reserved) < Max -> 97 | Tref = erlang:send_after(?RESERVATION_TIMEOUT, self(), {reservation_expired, Partition}), 98 | Reserved2 = [{Partition, Tref} | State#state.reservations], 99 | {reply, ok, State#state{reservations = Reserved2}}; 100 | true -> 101 | ?LOG_INFO("Node busy for partition ~p. running=~p reserved=~p max=~p", 102 | [Partition, Running, Reserved, Max]), 103 | {reply, busy, State} 104 | end; 105 | 106 | %% @hidden 107 | %% This message is a call to prevent unreserve/reserve races, as well as 108 | %% detect failed processes. 109 | handle_call({unreserve, Partition}, _From, State) -> 110 | Reserved2 = cancel_reservation_timeout(Partition, State#state.reservations), 111 | {reply, ok, State#state{reservations = Reserved2}}; 112 | 113 | handle_call(_Request, _From, State) -> 114 | {reply, ok, State}. 115 | 116 | 117 | %% @hidden 118 | handle_cast({claim_reservation, Partition}, State) -> 119 | Reserved2 = cancel_reservation_timeout(Partition, State#state.reservations), 120 | {noreply, State#state{reservations = Reserved2}}; 121 | 122 | handle_cast(_Msg, State) -> 123 | {noreply, State}. 124 | 125 | 126 | %% @hidden 127 | handle_info({reservation_expired, Partition}, State) -> 128 | Reserved2 = cancel_reservation_timeout(Partition, State#state.reservations), 129 | {noreply, State#state{reservations = Reserved2}}; 130 | 131 | handle_info(_Info, State) -> 132 | {noreply, State}. 133 | 134 | 135 | %% @hidden 136 | terminate(_Reason, _State) -> 137 | ok. 138 | 139 | 140 | %% @hidden 141 | code_change(_OldVsn, State, _Extra) -> 142 | {ok, State}. 143 | 144 | %% ------------------------------------------------------------------ 145 | %% Internal Function Definitions 146 | %% ------------------------------------------------------------------ 147 | 148 | get_partition_node(Partition) -> 149 | {ok, Ring} = riak_core_ring_manager:get_my_ring(), 150 | Owners = riak_core_ring:all_owners(Ring), 151 | proplists:get_value(Partition, Owners). 152 | 153 | cancel_reservation_timeout(Partition, Reserved) -> 154 | case proplists:get_value(Partition, Reserved) of 155 | undefined -> 156 | Reserved; 157 | Tref -> 158 | _ = erlang:cancel_timer(Tref), 159 | proplists:delete(Partition, Reserved) 160 | end. 161 | 162 | -------------------------------------------------------------------------------- /src/riak_repl2_fscoordinator_serv_sup.erl: -------------------------------------------------------------------------------- 1 | -module(riak_repl2_fscoordinator_serv_sup). 2 | -behavior(supervisor). 3 | 4 | -export([init/1]). 5 | 6 | -export([start_link/0, start_child/4, started/0, started/1]). 7 | 8 | start_link() -> 9 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 10 | 11 | start_child(Socket, Transport, Proto, Props) -> 12 | supervisor:start_child(?MODULE, [Socket, Transport, Proto, Props]). 13 | 14 | init(_) -> 15 | ChildSpec = {id, {riak_repl2_fscoordinator_serv, start_link, []}, 16 | temporary, brutal_kill, worker, [riak_repl2_fscoordinator_serv]}, 17 | {ok, {{simple_one_for_one, 10, 10}, [ChildSpec]}}. 18 | 19 | started() -> 20 | [{Remote, Pid} || {Remote, Pid, _, _} <- 21 | supervisor:which_children(?MODULE), is_pid(Pid)]. 22 | 23 | started(Node) -> 24 | [{Remote, Pid} || {Remote, Pid, _, _} <- 25 | supervisor:which_children({?MODULE, Node}), is_pid(Pid)]. 26 | 27 | -------------------------------------------------------------------------------- /src/riak_repl2_fscoordinator_sup.erl: -------------------------------------------------------------------------------- 1 | %% @doc simple supervisor for the fscoordinator, starting/stopping it 2 | %% in response to leader changes. 3 | -module(riak_repl2_fscoordinator_sup). 4 | 5 | -include_lib("kernel/include/logger.hrl"). 6 | 7 | -export([start_link/0, set_leader/2, start_coord/2, stop_coord/2, 8 | started/0, started/1, coord_for_cluster/1]). 9 | -export([init/1]). 10 | 11 | -define(SHUTDOWN, 5000). 12 | 13 | start_link() -> 14 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 15 | 16 | set_leader(Node, _Pid) -> 17 | case node() of 18 | Node -> 19 | {ok, Ring} = riak_core_ring_manager:get_my_ring(), 20 | Fullsyncs = riak_repl_ring:fs_enabled(Ring), 21 | [start_coord(node(), Remote) || Remote <- Fullsyncs]; 22 | _ -> 23 | [stop_coord(node(), Remote) || Remote <- started()] 24 | end. 25 | 26 | start_coord(Node, Remote) -> 27 | ?LOG_INFO("Starting replication coordination ~p", [Remote]), 28 | Childspec = make_remote(Remote), 29 | supervisor:start_child({?MODULE, Node}, Childspec). 30 | 31 | stop_coord(Node, Remote) -> 32 | ?LOG_INFO("Stopping replication coordination ~p", [Remote]), 33 | _ = supervisor:terminate_child({?MODULE, Node}, Remote), 34 | _ = supervisor:delete_child({?MODULE, Node}, Remote). 35 | 36 | started() -> 37 | [{Remote, Pid} || {Remote, Pid, _, _} <- 38 | supervisor:which_children(?MODULE), is_pid(Pid)]. 39 | 40 | started(Node) -> 41 | [{Remote, Pid} || {Remote, Pid, _, _} <- 42 | supervisor:which_children({?MODULE, Node}), is_pid(Pid)]. 43 | 44 | coord_for_cluster(Cluster) -> 45 | Coords = [{Remote, Pid} || {Remote, Pid, _, _} <- supervisor:which_children(?MODULE), 46 | is_pid(Pid), Remote == Cluster], 47 | case Coords of 48 | [] -> undefined; 49 | [{_,CoordPid}|_] -> CoordPid 50 | end. 51 | 52 | %% @private 53 | init(_) -> 54 | {ok, {{one_for_one, 10, 5}, []}}. 55 | 56 | make_remote(Remote) -> 57 | {Remote, {riak_repl2_fscoordinator, start_link, [Remote]}, 58 | transient, ?SHUTDOWN, worker, [riak_repl2, fscoordinator]}. 59 | -------------------------------------------------------------------------------- /src/riak_repl2_fssink_pool.erl: -------------------------------------------------------------------------------- 1 | %% Fullsync pool to be shared by all sinks. Globally bounded in size in case multiple 2 | %% fullsyncs are running. 3 | 4 | -module(riak_repl2_fssink_pool). 5 | -export([start_link/0, status/0, bin_put/1]). 6 | 7 | start_link() -> 8 | MinPool = app_helper:get_env(riak_repl, fssink_min_workers, 5), 9 | MaxPool = app_helper:get_env(riak_repl, fssink_max_workers, 100), 10 | PoolArgs = [{name, {local, ?MODULE}}, 11 | {worker_module, riak_repl_fullsync_worker}, 12 | {worker_args, []}, 13 | {size, MinPool}, {max_overflow, MaxPool}], 14 | poolboy:start_link(PoolArgs). 15 | 16 | %% @doc Return the poolboy status 17 | status() -> 18 | {StateName, WorkerQueueLen, Overflow, NumMonitors} = poolboy:status(?MODULE), 19 | [{statename, StateName}, 20 | {worker_queue_len, WorkerQueueLen}, 21 | {overflow, Overflow}, 22 | {num_monitors, NumMonitors}]. 23 | 24 | %% @doc Send a replication wire-encoded binary to the worker pool 25 | %% for running a put against. No guarantees of completion. 26 | bin_put(BinObj) -> 27 | Pid = poolboy:checkout(?MODULE, true, infinity), 28 | riak_repl_fullsync_worker:do_binput(Pid, BinObj, ?MODULE). 29 | -------------------------------------------------------------------------------- /src/riak_repl2_fssink_sup.erl: -------------------------------------------------------------------------------- 1 | %% Riak EnterpriseDS 2 | %% Copyright 2007-2012 Basho Technologies, Inc. All Rights Reserved. 3 | -module(riak_repl2_fssink_sup). 4 | -behaviour(supervisor). 5 | -export([start_link/0, start_child/4, started/0]). 6 | -export([init/1]). 7 | 8 | -define(SHUTDOWN, 5000). % how long to give rtsource processes to persist queue/shutdown 9 | 10 | start_link() -> 11 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 12 | 13 | start_child(Socket, Transport, Proto, Props) -> 14 | supervisor:start_child(?MODULE, [Socket, Transport, Proto, Props]). 15 | 16 | started() -> 17 | [Pid || {_, Pid, _, _} <- supervisor:which_children(?MODULE)]. 18 | 19 | %% @private 20 | init([]) -> 21 | ChildSpec = {undefined, {riak_repl2_fssink, start_link, []}, 22 | temporary, 5000, worker, [riak_repl2_fssink]}, 23 | {ok, {{simple_one_for_one, 10, 10}, [ChildSpec]}}. 24 | 25 | -------------------------------------------------------------------------------- /src/riak_repl2_fssource_sup.erl: -------------------------------------------------------------------------------- 1 | %% Riak EnterpriseDS 2 | %% Copyright 2007-2012 Basho Technologies, Inc. All Rights Reserved. 3 | -module(riak_repl2_fssource_sup). 4 | -behaviour(supervisor). 5 | 6 | -include_lib("kernel/include/logger.hrl"). 7 | 8 | -export([start_link/0, enable/3, disable/2, enabled/0, enabled/1]). 9 | -export([init/1]). 10 | 11 | -define(SHUTDOWN, 5000). % how long to give rtsource processes to persist queue/shutdown 12 | 13 | start_link() -> 14 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 15 | 16 | %%TODO: Rename enable/disable something better - start/stop is a bit overloaded 17 | enable(Node, Partition, IP) -> 18 | ?LOG_INFO("Starting replication fullsync source for ~p from ~p to ~p", 19 | [Partition, Node, IP]), 20 | ChildSpec = make_childspec(Partition, IP), 21 | supervisor:start_child({?MODULE, Node}, ChildSpec). 22 | 23 | disable(Node, Partition) -> 24 | ?LOG_INFO("Stopping replication fullsync source for ~p", [Partition]), 25 | _ = supervisor:terminate_child({?MODULE, Node}, Partition), 26 | _ = supervisor:delete_child({?MODULE, Node}, Partition). 27 | 28 | enabled() -> 29 | [{Remote, Pid} || {Remote, Pid, _, _} <- 30 | supervisor:which_children(?MODULE), is_pid(Pid)]. 31 | 32 | enabled(Node) -> 33 | [{Partition, Pid} || {Partition, Pid, _, _} <- 34 | supervisor:which_children({?MODULE, Node}), is_pid(Pid)]. 35 | 36 | %% @private 37 | init([]) -> 38 | {ok, {{one_for_one, 10, 10}, []}}. 39 | 40 | make_childspec(Partition, IP) -> 41 | {Partition, {riak_repl2_fssource, start_link, [Partition, IP]}, 42 | temporary, ?SHUTDOWN, worker, [riak_repl2_fssource]}. 43 | -------------------------------------------------------------------------------- /src/riak_repl2_pg.erl: -------------------------------------------------------------------------------- 1 | %% Riak EnterpriseDS 2 | %% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. 3 | %% 4 | %% Riak MDC "Advanced Mode" Proxy-Get 5 | 6 | -module(riak_repl2_pg). 7 | 8 | -include_lib("kernel/include/logger.hrl"). 9 | 10 | %% @doc Riak CS Proxy-get 11 | %% 12 | %% 13 | -export([start_link/0, status/0]). 14 | -export([enable/1, disable/1, enabled/0]). 15 | -export([register_remote_locator/0, ensure_pg/1]). 16 | 17 | %% gen_server callbacks 18 | -export([init/1, handle_call/3, handle_cast/2, handle_info/2, 19 | terminate/2, code_change/3]). 20 | 21 | -define(SERVER, ?MODULE). 22 | -record(state, {sinks = []}). 23 | 24 | %% API - is there any state? who watches ring events? 25 | start_link() -> 26 | gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). 27 | 28 | status() -> 29 | LeaderNode = riak_repl2_leader:leader_node(), 30 | case LeaderNode of 31 | undefined -> 32 | [{proxy_get, []}]; 33 | _ -> 34 | ReqStats = try riak_repl2_pg_block_requester_sup:started(LeaderNode) of 35 | [] -> 36 | []; 37 | PGRs -> 38 | [riak_repl2_pg_block_requester:status(Pid) || {_Remote, Pid} <- PGRs] 39 | catch 40 | _:_ -> 41 | [] 42 | end, 43 | ProStats = try riak_repl2_pg_block_provider_sup:enabled(LeaderNode) of 44 | [] -> 45 | []; 46 | PGPs -> 47 | [{Remote, riak_repl2_pg_block_provider:status(Pid)} || {Remote, Pid} <- PGPs] 48 | catch 49 | _:_ -> 50 | [] 51 | end, 52 | [{proxy_get,[{requester,ReqStats}, {provider, ProStats}]}] 53 | end. 54 | 55 | enabled() -> 56 | {ok, Ring} = riak_core_ring_manager:get_my_ring(), 57 | riak_repl_ring:pg_enabled(Ring). 58 | 59 | enable(Remote) -> 60 | do_ring_trans(fun riak_repl_ring:pg_enable_trans/2, Remote). 61 | 62 | disable(Remote) -> 63 | F = fun(Ring, Remote1) -> 64 | R2 = case riak_repl_ring:pg_disable_trans(Ring, Remote1) of 65 | {new_ring, R1} -> 66 | R1; 67 | {ignore, _} -> 68 | Ring 69 | end, 70 | riak_repl_ring:pg_disable_trans(R2, Remote1) 71 | end, 72 | do_ring_trans(F, Remote). 73 | 74 | ensure_pg(WantEnabled0) -> 75 | WantEnabled = lists:usort(WantEnabled0), 76 | Enabled = [Remote || {Remote, _Pid} <- riak_repl2_pg_block_provider_sup:enabled()], 77 | ToEnable = WantEnabled -- Enabled, 78 | ToDisable = Enabled -- WantEnabled, 79 | 80 | case ToEnable ++ ToDisable of 81 | [] -> 82 | []; 83 | _ -> 84 | ?LOG_DEBUG("proxy_get ToEnable : ~p", [ToEnable]), 85 | ?LOG_DEBUG("proxy_get ToDisable: ~p", [ToDisable]), 86 | _ = [riak_repl2_pg_block_provider_sup:enable(Remote) || 87 | Remote <- ToEnable], 88 | _ = [riak_repl2_pg_block_provider_sup:disable(Remote) || 89 | Remote<- ToDisable], 90 | [{enabled, ToEnable}, 91 | {disabled, ToDisable}] 92 | end. 93 | 94 | register_remote_locator() -> 95 | Locator = fun(Name, _Policy) -> 96 | riak_core_cluster_mgr:get_ipaddrs_of_cluster(Name) 97 | end, 98 | ok = riak_core_connection_mgr:register_locator(proxy_get, Locator). 99 | 100 | %% gen_server callbacks 101 | init([]) -> 102 | {ok, #state{}}. 103 | 104 | handle_call(status, _From, State) -> 105 | Status = {proxy_get, not_implemented}, 106 | {reply, Status, State}. 107 | 108 | handle_cast(Msg, State) -> 109 | ?LOG_WARNING("Proxy-get received an unknown cast: ~p", [Msg]), 110 | {noreply, State}. 111 | 112 | handle_info({'DOWN', _MRef, process, SinkPid, _Reason}, 113 | State = #state{sinks = Sinks}) -> 114 | Sinks2 = Sinks -- [SinkPid], 115 | {noreply, State#state{sinks = Sinks2}}; 116 | handle_info(Msg, State) -> 117 | ?LOG_WARNING("unhandled message - e.g. timed out status result: ~p", Msg), 118 | {noreply, State}. 119 | 120 | terminate(_Reason, _State) -> 121 | ok. 122 | 123 | code_change(_OldVsn, State, _Extra) -> 124 | {ok, State}. 125 | 126 | 127 | do_ring_trans(F, A) -> 128 | case riak_core_ring_manager:ring_trans(F, A) of 129 | {ok, _} -> 130 | ok; 131 | ER -> 132 | ER 133 | end. 134 | -------------------------------------------------------------------------------- /src/riak_repl2_pg_block_provider.erl: -------------------------------------------------------------------------------- 1 | %% Riak EnterpriseDS 2 | %% Copyright 2007-2013 Basho Technologies, Inc. All Rights Reserved. 3 | %% 4 | %% block_provider services proxy_get requests originating from a 5 | %% block_requester (which runs on the *SINK* cluster) to a *SOURCE* cluster. 6 | %% 7 | -module(riak_repl2_pg_block_provider). 8 | -include("riak_repl.hrl"). 9 | 10 | -include_lib("kernel/include/logger.hrl"). 11 | 12 | -behaviour(gen_server). 13 | %% API 14 | -export([start_link/1, connected/6, connect_failed/3]). 15 | 16 | %% gen_server callbacks 17 | -export([init/1, handle_call/3, handle_cast/2, handle_info/2, 18 | terminate/2, code_change/3, status/1, status/2]). 19 | 20 | %% send a message every KEEPALIVE milliseconds to make sure the service is running on the sink 21 | -define(KEEPALIVE, 60000). 22 | 23 | -record(state, 24 | { 25 | transport, 26 | socket, 27 | ip, 28 | other_cluster, 29 | connection_ref, 30 | worker, 31 | client, 32 | proxy_gets_provided = 0 33 | }). 34 | 35 | start_link(Cluster) -> 36 | gen_server:start_link(?MODULE, Cluster, []). 37 | 38 | status(Pid) -> 39 | status(Pid, infinity). 40 | 41 | status(Pid, Timeout) -> 42 | gen_server:call(Pid, status, Timeout). 43 | 44 | %% connection manager callbacks 45 | connected(Socket, Transport, Endpoint, Proto, Pid, Props) -> 46 | Transport:controlling_process(Socket, Pid), 47 | gen_server:call(Pid, 48 | {connected, Socket, Transport, Endpoint, Proto, Props}, ?LONG_TIMEOUT). 49 | 50 | connect_failed(_ClientProto, Reason, Pid) -> 51 | gen_server:cast(Pid, {connect_failed, self(), Reason}). 52 | 53 | 54 | init(Cluster) -> 55 | TcpOptions = [{keepalive, true}, 56 | {nodelay, true}, 57 | {packet, 4}, 58 | {active, false}], 59 | ClientSpec = {{proxy_get,[{1,0}]}, {TcpOptions, ?MODULE, self()}}, 60 | ?LOG_INFO("proxy_get connecting to remote ~p", [Cluster]), 61 | case riak_core_connection_mgr:connect({proxy_get, Cluster}, ClientSpec) of 62 | {ok, Ref} -> 63 | ?LOG_DEBUG("proxy_get connection ref ~p", [Ref]), 64 | {ok, #state{other_cluster = Cluster, connection_ref = Ref}}; 65 | {error, Reason}-> 66 | ?LOG_WARNING("Error connecting to remote"), 67 | {stop, Reason} 68 | end. 69 | 70 | handle_call({connected, Socket, Transport, _Endpoint, _Proto, Props}, _From, 71 | State=#state{other_cluster=OtherCluster}) -> 72 | Cluster = proplists:get_value(clustername, Props), 73 | ?LOG_DEBUG("proxy_get connected to ~p", [OtherCluster]), 74 | 75 | SocketTag = riak_repl_util:generate_socket_tag("pg_provider", Transport, Socket), 76 | ?LOG_DEBUG("Keeping stats for " ++ SocketTag), 77 | riak_core_tcp_mon:monitor(Socket, {?TCP_MON_PROXYGET_APP, source, 78 | SocketTag}, Transport), 79 | Transport:setopts(Socket, [{active, once}]), 80 | {ok, Client} = riak:local_client(), 81 | State2 = State#state{ 82 | transport=Transport, 83 | socket=Socket, 84 | other_cluster=Cluster, 85 | client=Client 86 | }, 87 | _ = keepalive_timer(), 88 | {reply, ok, State2}; 89 | handle_call(status, _From, State=#state{socket=Socket, 90 | proxy_gets_provided=PGCount}) -> 91 | SocketStats = riak_core_tcp_mon:socket_status(Socket), 92 | FormattedSS = {socket, 93 | riak_core_tcp_mon:format_socket_stats(SocketStats,[])}, 94 | 95 | Status = [ {provider_count, PGCount}, 96 | FormattedSS ], 97 | {reply, Status, State}; 98 | handle_call(_Msg, _From, State) -> 99 | {reply, ok, State}. 100 | 101 | 102 | handle_cast({connect_failed, _Pid, Reason}, 103 | State = #state{other_cluster = Cluster}) -> 104 | ?LOG_WARNING("proxy_get connection to cluster ~p failed ~p", 105 | [Cluster, Reason]), 106 | {stop, restart_it, State}; 107 | handle_cast(_Msg, State) -> 108 | {noreply, State}. 109 | 110 | handle_info(keepalive, State=#state{socket=Socket, transport=Transport}) -> 111 | Data = term_to_binary(stay_awake), 112 | Transport:send(Socket, Data), 113 | _ = keepalive_timer(), 114 | {noreply, State}; 115 | handle_info({tcp_closed, Socket}, State=#state{socket=Socket}) -> 116 | ?LOG_INFO("Connection for proxy_get ~p closed", [State#state.other_cluster]), 117 | {stop, socket_closed, State}; 118 | handle_info({tcp_error, _Socket, Reason}, State) -> 119 | ?LOG_ERROR("Connection for proxy_get ~p closed unexpectedly: ~p", 120 | [State#state.other_cluster, Reason]), 121 | {stop, socket_closed, State}; 122 | handle_info({ssl_closed, Socket}, State=#state{socket=Socket}) -> 123 | ?LOG_INFO("Connection for proxy_get ~p closed", [State#state.other_cluster]), 124 | {stop, socket_closed, State}; 125 | handle_info({ssl_error, _Socket, Reason}, State) -> 126 | ?LOG_ERROR("Connection for proxy_get ~p closed unexpectedly: ~p", 127 | [State#state.other_cluster, Reason]), 128 | {stop, socket_closed, State}; 129 | handle_info({Proto, Socket, Data}, 130 | State0=#state{socket=Socket,transport=Transport}) 131 | when Proto==tcp; Proto==ssl -> 132 | Transport:setopts(Socket, [{active, once}]), 133 | Msg = binary_to_term(Data), 134 | handle_msg(Msg, State0); 135 | handle_info(_Msg, State) -> 136 | {noreply, State}. 137 | 138 | handle_msg(get_cluster_info, State=#state{transport=Transport, socket=Socket}) -> 139 | ThisClusterName = riak_core_connection:symbolic_clustername(), 140 | {ok, Ring} = riak_core_ring_manager:get_my_ring(), 141 | ClusterID = riak_core_ring:cluster_name(Ring), 142 | ?LOG_DEBUG("Cluster ID=~p, Cluster Name = ~p",[ClusterID, ThisClusterName]), 143 | Data = term_to_binary({get_cluster_info_resp, ClusterID, ThisClusterName}), 144 | Transport:send(Socket, Data), 145 | {noreply, State}; 146 | handle_msg({proxy_get, Ref, Bucket, Key, Options}, 147 | State=#state{transport=Transport, socket=Socket, 148 | proxy_gets_provided=PGCount}) -> 149 | ?LOG_DEBUG("Got proxy_get for ~p:~p", [Bucket, Key]), 150 | C = State#state.client, 151 | Res = riak_client:get(Bucket, Key, Options, C), 152 | Data = term_to_binary({proxy_get_resp, Ref, Res}), 153 | Transport:send(Socket, Data), 154 | {noreply, State#state{proxy_gets_provided=PGCount+1}}. 155 | 156 | terminate(_Reason, #state{}) -> 157 | ok. 158 | 159 | code_change(_OldVsn, State, _Extra) -> 160 | {ok, State}. 161 | 162 | keepalive_timer() -> 163 | erlang:send_after(?KEEPALIVE, self(), keepalive). 164 | -------------------------------------------------------------------------------- /src/riak_repl2_pg_block_provider_sup.erl: -------------------------------------------------------------------------------- 1 | %% Riak EnterpriseDS 2 | %% Copyright 2007-2013 Basho Technologies, Inc. All Rights Reserved. 3 | -module(riak_repl2_pg_block_provider_sup). 4 | -behaviour(supervisor). 5 | -export([start_link/0, enable/1, enabled/0, enabled/1, disable/1]). 6 | -export([init/1]). 7 | 8 | -define(SHUTDOWN, 5000). 9 | 10 | start_link() -> 11 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 12 | 13 | enable(Remote) -> 14 | ChildSpec = make_remote(Remote), 15 | supervisor:start_child(?MODULE, ChildSpec). 16 | 17 | disable(Remote) -> 18 | _ = supervisor:terminate_child(?MODULE, Remote), 19 | _ = supervisor:delete_child(?MODULE, Remote). 20 | 21 | enabled() -> 22 | [{Remote, Pid} || {Remote, Pid, _, _} <- 23 | supervisor:which_children(?MODULE), is_pid(Pid)]. 24 | 25 | enabled(Node) -> 26 | [{Remote, Pid} || {Remote, Pid, _, _} <- 27 | supervisor:which_children({?MODULE, Node}), is_pid(Pid)]. 28 | 29 | %% @private 30 | init([]) -> 31 | riak_repl2_pg:register_remote_locator(), 32 | {ok, Ring} = riak_core_ring_manager:get_raw_ring(), 33 | Remotes = riak_repl_ring:pg_enabled(Ring), 34 | Children = [make_remote(Remote) || Remote <- Remotes], 35 | {ok, {{one_for_one, 10, 10}, Children}}. 36 | 37 | make_remote(Remote) -> 38 | {Remote, {riak_repl2_pg_block_provider, start_link, [Remote]}, 39 | transient, ?SHUTDOWN, worker, [riak_repl2_pg_block_provider]}. 40 | 41 | -------------------------------------------------------------------------------- /src/riak_repl2_pg_block_requester_sup.erl: -------------------------------------------------------------------------------- 1 | %% Riak EnterpriseDS 2 | %% Copyright 2007-2013 Basho Technologies, Inc. All Rights Reserved. 3 | -module(riak_repl2_pg_block_requester_sup). 4 | -behaviour(supervisor). 5 | -export([start_link/0, start_child/4, started/0, started/1]). 6 | -export([init/1]). 7 | 8 | start_link() -> 9 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 10 | 11 | start_child(Socket, Transport, Proto, Props) -> 12 | supervisor:start_child(?MODULE, [Socket, Transport, Proto, Props]). 13 | 14 | started() -> 15 | [Pid || {_, Pid, _, _} <- supervisor:which_children(?MODULE)]. 16 | 17 | started(Node) -> 18 | [{Remote, Pid} || {Remote, Pid, _, _} <- 19 | supervisor:which_children({?MODULE, Node}), is_pid(Pid)]. 20 | 21 | %% @private 22 | init([]) -> 23 | ChildSpec = {riak_repl2_pg_block_requester, {riak_repl2_pg_block_requester, start_link, []}, 24 | temporary, 5000, worker, [riak_repl2_pg_block_requester]}, 25 | {ok, {{simple_one_for_one, 10, 10}, [ChildSpec]}}. 26 | 27 | -------------------------------------------------------------------------------- /src/riak_repl2_pg_proxy.erl: -------------------------------------------------------------------------------- 1 | %% Riak EnterpriseDS 2 | %% Copyright 2007-2013 Basho Technologies, Inc. All Rights Reserved. 3 | %% 4 | %% pg_proxy keeps track of which node is servicing proxy_get requests 5 | %% in the cluster. A client can always make requests to the leader 6 | %% pg_proxy, which will then be routed to the appropriate node in the 7 | %% cluster. 8 | %% 9 | 10 | -module(riak_repl2_pg_proxy). 11 | 12 | -behaviour(gen_server). 13 | 14 | %% API 15 | -export([start_link/1]). 16 | 17 | %% gen_server callbacks 18 | -export([init/1, handle_call/3, handle_cast/2, handle_info/2, 19 | terminate/2, code_change/3, proxy_get/4]). 20 | 21 | -include("riak_repl.hrl"). 22 | 23 | -include_lib("kernel/include/logger.hrl"). 24 | 25 | -define(SERVER, ?MODULE). 26 | 27 | -record(state, { 28 | pg_pids = [] 29 | }). 30 | 31 | %%%=================================================================== 32 | %%% API 33 | %%%=================================================================== 34 | proxy_get(Pid, Bucket, Key, Options) -> 35 | gen_server:call(Pid, {proxy_get, Bucket, Key, Options}, ?LONG_TIMEOUT). 36 | 37 | %%-------------------------------------------------------------------- 38 | %% @doc 39 | %% Starts the server 40 | %% 41 | %% @end 42 | %%-------------------------------------------------------------------- 43 | -spec start_link(ProxyName :: string()) -> {'ok', pid()}. 44 | start_link(ProxyName) -> 45 | gen_server:start_link(?MODULE, ProxyName, []). 46 | 47 | %%%=================================================================== 48 | %%% Gen_server callbacks 49 | %%%=================================================================== 50 | 51 | init(ProxyName) -> 52 | ?LOG_DEBUG("Registering pg_proxy ~p", [ProxyName]), 53 | erlang:register(ProxyName, self()), 54 | {ok, #state{}}. 55 | 56 | handle_call({proxy_get, Bucket, Key, GetOptions}, _From, 57 | #state{pg_pids=RequesterPids} = State) -> 58 | case RequesterPids of 59 | [] -> 60 | ?LOG_WARNING("No proxy_get node registered"), 61 | {reply, {error, no_proxy_get_node}, State}; 62 | [{_RNode, RPid, _} | T] -> 63 | try gen_server:call(RPid, {proxy_get, Bucket, Key, GetOptions}, ?LONG_TIMEOUT) of 64 | Result -> 65 | {reply, Result, State} 66 | catch 67 | %% remove this bad pid from the list and try another 68 | exit:{noproc, _} -> 69 | handle_call({proxy_get, Bucket, Key, GetOptions}, _From, 70 | State#state{pg_pids=T}); 71 | exit:{{nodedown, _}, _} -> 72 | handle_call({proxy_get, Bucket, Key, GetOptions}, _From, 73 | State#state{pg_pids=T}) 74 | end 75 | end; 76 | 77 | handle_call({register, ClusterName, RequesterNode, RequesterPid}, 78 | _From, State = #state{pg_pids = RequesterPids}) -> 79 | ?LOG_INFO("registered node for cluster name ~p ~p ~p", [ClusterName, 80 | RequesterNode, 81 | RequesterPid]), 82 | Monitor = erlang:monitor(process, RequesterPid), 83 | NewState = State#state{pg_pids = [{RequesterNode, RequesterPid, Monitor} | 84 | RequesterPids]}, 85 | {reply, ok, NewState}. 86 | 87 | handle_info({'DOWN', MRef, process, _Pid, _Reason}, State = 88 | #state{pg_pids=RequesterPids}) -> 89 | NewRequesterPids = [ {RNode, RPid, RMon} || 90 | {RNode,RPid,RMon} <- RequesterPids, 91 | RMon /= MRef], 92 | {noreply, State#state{pg_pids=NewRequesterPids}}; 93 | handle_info(_Info, State) -> 94 | {noreply, State}. 95 | 96 | handle_cast(_Msg, State) -> 97 | {noreply, State}. 98 | 99 | terminate(_Reason, _State) -> 100 | ok. 101 | 102 | code_change(_OldVsn, State, _Extra) -> 103 | {ok, State}. 104 | 105 | -------------------------------------------------------------------------------- /src/riak_repl2_pg_proxy_sup.erl: -------------------------------------------------------------------------------- 1 | %%%------------------------------------------------------------------- 2 | %%% Created : 21 Feb 2013 by Dave Parfitt 3 | %%%------------------------------------------------------------------- 4 | -module(riak_repl2_pg_proxy_sup). 5 | 6 | -behaviour(supervisor). 7 | 8 | -include_lib("kernel/include/logger.hrl"). 9 | 10 | %% API 11 | -export([start_link/0, set_leader/2, started/1, start_proxy/1, make_remote/1, stop_proxy/2]). 12 | 13 | %% Supervisor callbacks 14 | -export([init/1]). 15 | 16 | -define(SERVER, ?MODULE). 17 | -define(SHUTDOWN, 5000). 18 | 19 | %%%=================================================================== 20 | %%% API functions 21 | %%%=================================================================== 22 | start_link() -> 23 | supervisor:start_link({local, ?SERVER}, ?MODULE, []). 24 | 25 | set_leader(Node, _Pid) -> 26 | case node() of 27 | Node -> ok; 28 | _ -> 29 | _ = [ begin 30 | _ = supervisor:terminate_child(?MODULE, Remote), 31 | _ = supervisor:delete_child(?MODULE, Remote) 32 | end 33 | || {Remote, Pid, _, _} <- 34 | supervisor:which_children(?MODULE), is_pid(Pid) 35 | ] 36 | end. 37 | 38 | start_proxy(Remote) -> 39 | ?LOG_DEBUG("Starting pg_proxy for ~p", [Remote]), 40 | Childspec = make_remote(Remote), 41 | supervisor:start_child({?MODULE, node()}, Childspec). 42 | 43 | stop_proxy(Node, Remote) -> 44 | ?LOG_DEBUG("Stopping pg_proxy for ~p", [Remote]), 45 | _ = supervisor:terminate_child({?MODULE, Node}, Remote), 46 | _ = supervisor:delete_child({?MODULE, Node}, Remote). 47 | 48 | started(Node) -> 49 | [{Remote, Pid} || {Remote, Pid, _, _} <- 50 | supervisor:which_children({?MODULE, Node}), is_pid(Pid)]. 51 | 52 | init(_) -> 53 | {ok, {{one_for_one, 10, 5}, []}}. 54 | 55 | make_remote(Remote) -> 56 | Name = riak_repl_util:make_pg_proxy_name(Remote), 57 | ?LOG_DEBUG("make_remote ~p", [Name]), 58 | {Name, {riak_repl2_pg_proxy, start_link, [Name]}, 59 | transient, ?SHUTDOWN, worker, [riak_repl2_pg_proxy, pg_proxy]}. 60 | 61 | -------------------------------------------------------------------------------- /src/riak_repl2_pg_sup.erl: -------------------------------------------------------------------------------- 1 | %% Riak EnterpriseDS 2 | %% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. 3 | -module(riak_repl2_pg_sup). 4 | -behaviour(supervisor). 5 | 6 | %% External exports 7 | -export([start_link/0]). 8 | %% supervisor callbacks 9 | -export([init/1]). 10 | 11 | %% @spec start_link() -> ServerRet 12 | %% @doc API for starting the supervisor. 13 | start_link() -> 14 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 15 | 16 | %% @spec init([]) -> SupervisorTree 17 | %% @doc supervisor callback. 18 | init([]) -> 19 | Processes = 20 | [ {riak_repl2_pg_block_provider_sup, 21 | { riak_repl2_pg_block_provider_sup, start_link, []}, 22 | permanent, infinity, supervisor, [ 23 | riak_repl_pg_block_provider_sup]}, 24 | 25 | {riak_repl2_pg_block_requester_sup, 26 | {riak_repl2_pg_block_requester_sup, start_link, []}, 27 | permanent, infinity, supervisor, 28 | [riak_repl2_pg_block_requester_sup]}, 29 | 30 | {riak_repl2_pg, {riak_repl2_pg, start_link, []}, 31 | permanent, 50000, worker, [riak_repl2_pg]} ], 32 | {ok, {{one_for_one, 9, 10}, Processes}}. 33 | -------------------------------------------------------------------------------- /src/riak_repl2_rt_sup.erl: -------------------------------------------------------------------------------- 1 | %% Riak EnterpriseDS 2 | %% Copyright (c) 2007-2012 Basho Technologies, Inc. All Rights Reserved. 3 | -module(riak_repl2_rt_sup). 4 | -behaviour(supervisor). 5 | 6 | %% External exports 7 | -export([start_link/0]). 8 | %% supervisor callbacks 9 | -export([init/1]). 10 | 11 | %% @spec start_link() -> ServerRet 12 | %% @doc API for starting the supervisor. 13 | start_link() -> 14 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 15 | 16 | %% @spec init([]) -> SupervisorTree 17 | %% @doc supervisor callback. 18 | init([]) -> 19 | Processes = 20 | [ {riak_repl2_rtsource_sup, {riak_repl2_rtsource_sup, start_link, []}, 21 | permanent, infinity, supervisor, [riak_repl2_rtsource_sup]}, 22 | {riak_repl2_rtsink_sup, {riak_repl2_rtsink_sup, start_link, []}, 23 | permanent, infinity, supervisor, [riak_repl2_rtink_sup]}, 24 | {riak_repl2_rt, {riak_repl2_rt, start_link, []}, 25 | permanent, 50000, worker, [riak_repl2_rt]} ], 26 | {ok, {{one_for_one, 9, 10}, Processes}}. 27 | 28 | -------------------------------------------------------------------------------- /src/riak_repl2_rtframe.erl: -------------------------------------------------------------------------------- 1 | %% API 2 | -module(riak_repl2_rtframe). 3 | -export([encode/2, decode/1]). 4 | 5 | -define(MSG_HEARTBEAT, 16#00). %% Heartbeat message 6 | -define(MSG_OBJECTS, 16#10). %% List of objects to write 7 | -define(MSG_ACK, 16#20). %% Ack 8 | -define(MSG_OBJECTS_AND_META, 16#30). %% List of objects and rt meta data 9 | 10 | %% Build an IOlist suitable for sending over a socket 11 | encode(Type, Payload) -> 12 | IOL = encode_payload(Type, Payload), 13 | Size = iolist_size(IOL), 14 | [<> | IOL]. 15 | 16 | encode_payload(objects_and_meta, {Seq, BinObjs, Meta}) when is_binary(BinObjs) -> 17 | BinsAndMeta = {BinObjs, Meta}, 18 | BinsAndMetaBin = term_to_binary(BinsAndMeta), 19 | [?MSG_OBJECTS_AND_META, 20 | <>, 21 | BinsAndMetaBin]; 22 | encode_payload(objects, {Seq, BinObjs}) when is_binary(BinObjs) -> 23 | [?MSG_OBJECTS, 24 | <>, 25 | BinObjs]; 26 | encode_payload(ack, Seq) -> 27 | [?MSG_ACK, 28 | <>]; 29 | encode_payload(heartbeat, undefined) -> 30 | [?MSG_HEARTBEAT]. 31 | 32 | decode(<>) -> 35 | <> = Msg, 36 | {ok, decode_payload(MsgCode, Payload), Rest}; 37 | decode(<>) -> 38 | {ok, undefined, Rest}. 39 | 40 | decode_payload(?MSG_OBJECTS_AND_META, <>) -> 41 | {Bins, Meta} = binary_to_term(BinsAndMeta), 42 | {objects_and_meta, {Seq, Bins, Meta}}; 43 | decode_payload(?MSG_OBJECTS, <>) -> 44 | {objects, {Seq, BinObjs}}; 45 | decode_payload(?MSG_ACK, <>) -> 46 | {ack, Seq}; 47 | decode_payload(?MSG_HEARTBEAT, <<>>) -> 48 | heartbeat. 49 | -------------------------------------------------------------------------------- /src/riak_repl2_rtq_overload_counter.erl: -------------------------------------------------------------------------------- 1 | %% @doc Simply takes messages from rtq pushes that are dropped and counts them. 2 | %% At a configurable interval it tells the rtq how many have been dropped since 3 | %% last interval. If nothing has been dropped, no message is sent. 4 | -module(riak_repl2_rtq_overload_counter). 5 | -behavior(gen_server). 6 | 7 | -include_lib("kernel/include/logger.hrl"). 8 | 9 | -define(DEFAULT_INTERVAL, 5000). 10 | 11 | -record(state, { 12 | % number of drops since last report 13 | drops = 0 :: non_neg_integer(), 14 | % how often (in milliseconds) to report drops to rtq. 15 | interval :: pos_integer(), 16 | % timer reference for interval 17 | timer 18 | }). 19 | 20 | -export([start_link/0, start_link/1, stop/0]). 21 | -export([drop/0]). 22 | -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, 23 | code_change/3]). 24 | 25 | %% API 26 | 27 | %% @doc Start linked and registered as module name with default options. 28 | start_link() -> 29 | SendInterval = app_helper:get_env(riak_repl, rtq_drop_report_interval, ?DEFAULT_INTERVAL), 30 | start_link([{report_interval, SendInterval}]). 31 | 32 | %% @doc Start linked and registered as the module name with the given options. 33 | start_link(Options) -> 34 | gen_server:start_link({local, ?MODULE}, ?MODULE, Options, []). 35 | 36 | %% @doc Stop the counter gracefully. 37 | stop() -> 38 | gen_server:cast(?MODULE, stop). 39 | 40 | %% @private 41 | drop() -> 42 | gen_server:cast(?MODULE, drop). 43 | 44 | %% gen_server 45 | 46 | %% @private 47 | init(Options) -> 48 | Report = proplists:get_value(report_interval, Options, ?DEFAULT_INTERVAL), 49 | {ok, #state{interval = Report}}. 50 | 51 | handle_call(_Msg, _From, State) -> 52 | {reply, {error, badcall}, State}. 53 | 54 | handle_cast(drop, #state{timer = undefined} = State) -> 55 | {ok, Timer} = timer:send_after(State#state.interval, report_drops), 56 | {noreply, dropped(State#state{timer = Timer})}; 57 | 58 | handle_cast(drop, State) -> 59 | {noreply, dropped(State)}; 60 | 61 | handle_cast(stop, State) -> 62 | {stop, normal, State}; 63 | 64 | handle_cast(_Msg, State) -> 65 | {noreply, State}. 66 | 67 | handle_info(report_drops, State) -> 68 | ?LOG_DEBUG("reporting drops: ~p", [State#state.drops]), 69 | riak_repl2_rtq:report_drops(State#state.drops), 70 | {noreply, State#state{drops = 0, timer = undefined}}; 71 | 72 | handle_info(_Msg, State) -> 73 | {noreply, State}. 74 | 75 | terminate(_Why, _State) -> 76 | ok. 77 | 78 | code_change(_Vsn, State, _Extra) -> 79 | {ok, State}. 80 | 81 | %% internal 82 | 83 | dropped(#state{drops = N} = State) -> 84 | State#state{drops = N + 1}. 85 | -------------------------------------------------------------------------------- /src/riak_repl2_rtq_proxy.erl: -------------------------------------------------------------------------------- 1 | %% Riak EnterpriseDS 2 | %% Copyright (c) 2007-2012 Basho Technologies, Inc. All Rights Reserved. 3 | -module(riak_repl2_rtq_proxy). 4 | 5 | %% @doc A proxy process that handles realtime messages received while and 6 | %% after the riak_repl application has shut down. This allows us to avoid 7 | %% dropping realtime messages around shutdown events. 8 | 9 | -behaviour(gen_server). 10 | 11 | -include_lib("kernel/include/logger.hrl"). 12 | 13 | %% API 14 | -export([start/0, start_link/0, push/2, push/3]). 15 | 16 | %% gen_server callbacks 17 | -export([init/1, 18 | handle_call/3, 19 | handle_cast/2, 20 | handle_info/2, 21 | terminate/2, 22 | code_change/3]). 23 | 24 | -record(state, {nodes=[], %% peer replication nodes 25 | versions=[], %% {node(), wire-version()} 26 | meta_support=[] :: [{node(), boolean()}], 27 | refs=[] :: [reference()] 28 | }). 29 | 30 | %%%=================================================================== 31 | %%% API 32 | %%%=================================================================== 33 | 34 | start() -> 35 | %% start under the kernel_safe_sup supervisor so we can block node 36 | %% shutdown if we need to do process any outstanding work 37 | LogSup = {?MODULE, {?MODULE, start_link, []}, permanent, 38 | 5000, worker, [?MODULE]}, 39 | supervisor:start_child(kernel_safe_sup, LogSup). 40 | 41 | start_link() -> 42 | gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). 43 | 44 | push(NumItems, Bin) -> 45 | push(NumItems, Bin, []). 46 | 47 | push(NumItems, Bin, Meta) -> 48 | gen_server:cast(?MODULE, {push, NumItems, Bin, Meta}). 49 | 50 | %%%=================================================================== 51 | %%% gen_server callbacks 52 | %%%=================================================================== 53 | 54 | init([]) -> 55 | %% trap exit so we can have terminate() called 56 | process_flag(trap_exit, true), 57 | Nodes = riak_repl_util:get_peer_repl_nodes(), 58 | Refs = [erlang:monitor(process, {riak_repl2_rtq, Node}) || Node <- Nodes], 59 | %% cache the supported wire format of peer nodes to avoid rcp calls later. 60 | Versions = get_peer_wire_versions(Nodes), 61 | Metas = get_peer_meta_support(Nodes), 62 | {ok, #state{nodes=Nodes, versions=Versions, meta_support = Metas, refs=Refs}}. 63 | 64 | handle_call(_Request, _From, State) -> 65 | Reply = ok, 66 | {reply, Reply, State}. 67 | 68 | % we got a push from an older process/node, so upgrade it and retry. 69 | handle_cast({push, NumItems, Bin}, State) -> 70 | handle_cast({push, NumItems, Bin, []}, State); 71 | 72 | handle_cast({push, NumItems, _Bin, _Meta}, State = #state{nodes=[]}) -> 73 | ?LOG_WARNING("No available nodes to proxy ~p objects to~n", [NumItems]), 74 | catch(riak_repl_stats:rt_source_errors()), 75 | {noreply, State}; 76 | handle_cast({push, NumItems, W1BinObjs, Meta}, State) -> 77 | %% push items to another node for queueing. If the other node does not speak binary 78 | %% object format, then downconvert the items (if needed) before pushing. 79 | [Node | Nodes] = State#state.nodes, 80 | PeerWireVer = wire_version_of_node(Node, State#state.versions), 81 | ?LOG_DEBUG("Proxying ~p items to ~p with wire version ~p", [NumItems, Node, PeerWireVer]), 82 | BinObjs = riak_repl_util:maybe_downconvert_binary_objs(W1BinObjs, PeerWireVer), 83 | case meta_support(Node, State#state.meta_support) of 84 | true -> 85 | gen_server:cast({riak_repl2_rtq, Node}, {push, NumItems, BinObjs, Meta}); 86 | false -> 87 | gen_server:cast({riak_repl2_rtq, Node}, {push, NumItems, BinObjs}) 88 | end, 89 | {noreply, State#state{nodes=Nodes ++ [Node]}}; 90 | handle_cast(_Msg, State) -> 91 | {noreply, State}. 92 | 93 | handle_info({'DOWN', _Ref, _, {riak_repl2_rtq, Node}, _}, State) -> 94 | ?LOG_INFO("rtq proxy target ~p is down", [Node]), 95 | {noreply, State#state{nodes=State#state.nodes -- [Node]}}; 96 | handle_info(_Info, State) -> 97 | {noreply, State}. 98 | 99 | terminate(_Reason, State) -> 100 | case State#state.nodes of 101 | [] -> 102 | ok; 103 | _ -> 104 | %% relay as much as we can, blocking shutdown 105 | flush_pending_pushes(State) 106 | end, 107 | ok. 108 | 109 | code_change(_OldVsn, State, _Extra) -> 110 | {ok, State}. 111 | 112 | flush_pending_pushes(State) -> 113 | receive 114 | {'$gen_cast', Msg} -> 115 | {noreply, NewState} = handle_cast(Msg, State), 116 | flush_pending_pushes(NewState) 117 | after 118 | 100 -> 119 | ok 120 | end. 121 | 122 | %% return tuple of node and it's supported wire version 123 | get_peer_wire_versions(Nodes) -> 124 | [ begin 125 | WireVer = riak_repl_util:peer_wire_format(Node), 126 | {Node, WireVer} 127 | end || Node <- Nodes]. 128 | 129 | get_peer_meta_support(Nodes) -> 130 | GetSupport = fun(Node) -> 131 | case riak_core_util:safe_rpc(Node, riak_core_capability, get, [{riak_repl, rtq_meta}, false]) of 132 | Bool when is_boolean(Bool) -> 133 | {Node, Bool}; 134 | LolWut -> 135 | ?LOG_WARNING("Could not get a definitive result when querying ~p about it's rtq_meta support: ~p", [Node, LolWut]), 136 | {Node, false} 137 | end 138 | end, 139 | lists:map(GetSupport, Nodes). 140 | 141 | wire_version_of_node(Node, Versions) -> 142 | case lists:keyfind(Node, 1, Versions) of 143 | false -> 144 | w0; 145 | {_Node, Ver} -> 146 | Ver 147 | end. 148 | 149 | meta_support(Node, Metas) -> 150 | case lists:keyfind(Node, 1, Metas) of 151 | {_Node, Val} -> 152 | Val; 153 | false -> 154 | false 155 | end. 156 | -------------------------------------------------------------------------------- /src/riak_repl2_rtsink_conn_sup.erl: -------------------------------------------------------------------------------- 1 | %% Riak EnterpriseDS 2 | %% Copyright 2007-2012 Basho Technologies, Inc. All Rights Reserved. 3 | -module(riak_repl2_rtsink_conn_sup). 4 | -behaviour(supervisor). 5 | -export([start_link/0, start_child/2, started/0]). 6 | -export([init/1]). 7 | 8 | -define(SHUTDOWN, 5000). % how long to give rtsource processes to persist queue/shutdown 9 | 10 | start_link() -> 11 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 12 | 13 | start_child(Proto, Remote) -> 14 | supervisor:start_child(?MODULE, [Proto, Remote]). 15 | 16 | started() -> 17 | [Pid || {_, Pid, _, _} <- supervisor:which_children(?MODULE)]. 18 | 19 | %% @private 20 | init([]) -> 21 | ChildSpec = {undefined, {riak_repl2_rtsink_conn, start_link, []}, 22 | temporary, 5000, worker, [riak_repl2_rtsink_conn]}, 23 | {ok, {{simple_one_for_one, 10, 10}, [ChildSpec]}}. 24 | 25 | -------------------------------------------------------------------------------- /src/riak_repl2_rtsink_helper.erl: -------------------------------------------------------------------------------- 1 | %% Riak EnterpriseDS 2 | %% Copyright (c) 2007-2012 Basho Technologies, Inc. All Rights Reserved. 3 | 4 | %% @doc Realtime replication sink module 5 | %% 6 | %% High level responsibility... 7 | %% consider moving out socket responsibilities to another process 8 | %% to keep this one responsive (but it would pretty much just do status) 9 | %% 10 | -module(riak_repl2_rtsink_helper). 11 | 12 | %% API 13 | -export([start_link/1, 14 | stop/1, 15 | write_objects/4]). 16 | 17 | %% gen_server callbacks 18 | -export([init/1, 19 | handle_call/3, 20 | handle_cast/2, 21 | handle_info/2, 22 | terminate/2, 23 | code_change/3]). 24 | 25 | -behavior(gen_server). 26 | 27 | -record(state, {parent %% Parent process 28 | }). 29 | 30 | start_link(Parent) -> 31 | gen_server:start_link(?MODULE, [Parent], []). 32 | 33 | stop(Pid) -> 34 | gen_server:call(Pid, stop, infinity). 35 | 36 | write_objects(Pid, BinObjs, DoneFun, Ver) -> 37 | gen_server:cast(Pid, {write_objects, BinObjs, DoneFun, Ver}). 38 | 39 | %% Callbacks 40 | init([Parent]) -> 41 | {ok, #state{parent = Parent}}. 42 | 43 | handle_call(stop, _From, State) -> 44 | {stop, normal, ok, State}. 45 | 46 | handle_cast({write_objects, BinObjs, DoneFun, Ver}, State) -> 47 | do_write_objects(BinObjs, DoneFun, Ver), 48 | {noreply, State}; 49 | handle_cast({unmonitor, Ref}, State) -> 50 | demonitor(Ref), 51 | {noreply, State}. 52 | 53 | handle_info({'DOWN', _MRef, process, _Pid, Reason}, State) 54 | when Reason == normal; Reason == shutdown -> 55 | {noreply, State}; 56 | handle_info({'DOWN', _MRef, process, Pid, Reason}, State) -> 57 | %% TODO: Log worker failure 58 | %% TODO: Needs graceful way to let rtsink know so it can die 59 | {stop, {worker_died, {Pid, Reason}}, State}. 60 | 61 | terminate(_Reason, _State) -> 62 | %% TODO: Consider trying to do something graceful with poolboy? 63 | ok. 64 | 65 | code_change(_OldVsn, State, _Extra) -> 66 | {ok, State}. 67 | 68 | %% Receive TCP data - decode framing and dispatch 69 | do_write_objects(BinObjs, DoneFun, Ver) -> 70 | Worker = poolboy:checkout(riak_repl2_rtsink_pool, true, infinity), 71 | MRef = monitor(process, Worker), 72 | Me = self(), 73 | WrapperFun = fun() -> DoneFun(), gen_server:cast(Me, {unmonitor, MRef}) end, 74 | ok = riak_repl_fullsync_worker:do_binputs(Worker, BinObjs, WrapperFun, 75 | riak_repl2_rtsink_pool, Ver). 76 | -------------------------------------------------------------------------------- /src/riak_repl2_rtsink_sup.erl: -------------------------------------------------------------------------------- 1 | %% Riak EnterpriseDS 2 | %% Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved. 3 | -module(riak_repl2_rtsink_sup). 4 | -author('Andy Gross '). 5 | -behaviour(supervisor). 6 | 7 | %% External exports 8 | -export([start_link/0]). 9 | %% supervisor callbacks 10 | -export([init/1]). 11 | 12 | %% @spec start_link() -> ServerRet 13 | %% @doc API for starting the supervisor. 14 | start_link() -> 15 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 16 | 17 | %% @spec init([]) -> SupervisorTree 18 | %% @doc supervisor callback. 19 | init([]) -> 20 | MinPool = app_helper:get_env(riak_repl, rtsink_min_workers, 5), 21 | MaxPool = app_helper:get_env(riak_repl, rtsink_max_workers, 100), 22 | PoolArgs = [{name, {local, riak_repl2_rtsink_pool}}, 23 | {worker_module, riak_repl_fullsync_worker}, 24 | {worker_args, []}, 25 | {size, MinPool}, {max_overflow, MaxPool}], 26 | Processes = 27 | [ 28 | {riak_repl2_rtsink_pool, {poolboy, start_link, [PoolArgs]}, 29 | permanent, 5000, worker, [poolboy]}, 30 | 31 | {riak_repl2_rtsink_conn_sup, {riak_repl2_rtsink_conn_sup, start_link, []}, 32 | permanent, infinity, supervisor, [riak_repl2_rtsink_conn_sup]} ], 33 | {ok, {{rest_for_one, 9, 10}, Processes}}. 34 | -------------------------------------------------------------------------------- /src/riak_repl2_rtsource_conn_sup.erl: -------------------------------------------------------------------------------- 1 | %% Riak EnterpriseDS 2 | %% Copyright 2007-2012 Basho Technologies, Inc. All Rights Reserved. 3 | -module(riak_repl2_rtsource_conn_sup). 4 | -behaviour(supervisor). 5 | 6 | -include_lib("kernel/include/logger.hrl"). 7 | 8 | -export([start_link/0, enable/1, disable/1, enabled/0]). 9 | -export([init/1]). 10 | 11 | -define(SHUTDOWN, 5000). % how long to give rtsource processes to persist queue/shutdown 12 | 13 | start_link() -> 14 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 15 | 16 | %%TODO: Rename enable/disable something better - start/stop is a bit overloaded 17 | enable(Remote) -> 18 | ?LOG_INFO("Starting replication realtime source ~p", [Remote]), 19 | ChildSpec = make_remote(Remote), 20 | supervisor:start_child(?MODULE, ChildSpec). 21 | 22 | disable(Remote) -> 23 | ?LOG_INFO("Stopping replication realtime source ~p", [Remote]), 24 | _ = supervisor:terminate_child(?MODULE, Remote), 25 | _ = supervisor:delete_child(?MODULE, Remote). 26 | 27 | enabled() -> 28 | [{Remote, Pid} || {Remote, Pid, _, _} <- supervisor:which_children(?MODULE), is_pid(Pid)]. 29 | 30 | %% @private 31 | init([]) -> 32 | %% TODO: Move before riak_repl2_rt_sup start 33 | %% once connmgr is started by core. Must be started/registered 34 | %% before sources are started. 35 | riak_repl2_rt:register_remote_locator(), 36 | 37 | {ok, Ring} = riak_core_ring_manager:get_raw_ring(), 38 | Remotes = riak_repl_ring:rt_started(Ring), 39 | Children = [make_remote(Remote) || Remote <- Remotes], 40 | {ok, {{one_for_one, 10, 10}, Children}}. 41 | 42 | make_remote(Remote) -> 43 | {Remote, {riak_repl2_rtsource_conn, start_link, [Remote]}, 44 | permanent, ?SHUTDOWN, worker, [riak_repl2_rtsource_conn]}. 45 | -------------------------------------------------------------------------------- /src/riak_repl2_rtsource_helper.erl: -------------------------------------------------------------------------------- 1 | %% Riak EnterpriseDS 2 | %% Copyright (c) 2007-2012 Basho Technologies, Inc. All Rights Reserved. 3 | -module(riak_repl2_rtsource_helper). 4 | 5 | %% @doc Realtime replication source helper 6 | %% 7 | %% High level responsibility... 8 | 9 | -behaviour(gen_server). 10 | %% API 11 | -export([start_link/4, 12 | stop/1, 13 | v1_ack/2, 14 | status/1, status/2, send_heartbeat/1]). 15 | 16 | -include_lib("kernel/include/logger.hrl"). 17 | 18 | -include("riak_repl.hrl"). 19 | 20 | -define(SERVER, ?MODULE). 21 | 22 | %% gen_server callbacks 23 | -export([init/1, handle_call/3, handle_cast/2, handle_info/2, 24 | terminate/2, code_change/3]). 25 | 26 | -record(state, {remote, % remote site name 27 | transport, % erlang module to use for transport 28 | socket, % socket to pass to transport 29 | proto, % protocol version negotiated 30 | deliver_fun,% Deliver function 31 | sent_seq, % last sequence sent 32 | v1_offset = 0, 33 | v1_seq_map = [], 34 | objects = 0}). % number of objects sent - really number of pulls as could be multiobj 35 | 36 | start_link(Remote, Transport, Socket, Version) -> 37 | gen_server:start_link(?MODULE, [Remote, Transport, Socket, Version], []). 38 | 39 | stop(Pid) -> 40 | gen_server:call(Pid, stop, ?LONG_TIMEOUT). 41 | 42 | %% @doc v1 sinks require fully sequential sequence numbers sent. The outgoing 43 | %% Seq's are munged, and thus must be munged back when the sink replies. 44 | v1_ack(Pid, Seq) -> 45 | gen_server:cast(Pid, {v1_ack, Seq}). 46 | 47 | status(Pid) -> 48 | status(Pid, app_helper:get_env(riak_repl, riak_repl2_rtsource_helper_status_to, ?LONG_TIMEOUT)). 49 | 50 | status(Pid, Timeout) -> 51 | gen_server:call(Pid, status, Timeout). 52 | 53 | send_heartbeat(Pid) -> 54 | %% Cast the heartbeat, do not want to block the rtsource process 55 | %% as it is responsible for checking heartbeat 56 | gen_server:cast(Pid, send_heartbeat). 57 | 58 | init([Remote, Transport, Socket, Version]) -> 59 | _ = riak_repl2_rtq:register(Remote), % re-register to reset stale deliverfun 60 | Me = self(), 61 | Deliver = fun(Result) -> gen_server:call(Me, {pull, Result}, infinity) end, 62 | State = #state{remote = Remote, transport = Transport, proto = Version, 63 | socket = Socket, deliver_fun = Deliver}, 64 | async_pull(State), 65 | {ok, State}. 66 | 67 | handle_call({pull, {error, Reason}}, _From, State) -> 68 | riak_repl_stats:rt_source_errors(), 69 | {stop, {queue_error, Reason}, ok, State}; 70 | handle_call({pull, {Seq, NumObjects, _BinObjs, _Meta} = Entry}, From, 71 | State = #state{transport = T, socket = S, objects = Objects}) -> 72 | %% unblock the rtq as fast as possible 73 | gen_server:reply(From, ok), 74 | State2 = maybe_send(T, S, Entry, State), 75 | async_pull(State2), 76 | {noreply, State2#state{sent_seq = Seq, objects = Objects + NumObjects}}; 77 | handle_call(stop, _From, State) -> 78 | {stop, normal, ok, State}; 79 | handle_call(status, _From, State = 80 | #state{sent_seq = SentSeq, objects = Objects}) -> 81 | {reply, [{sent_seq, SentSeq}, 82 | {objects, Objects}], State}. 83 | 84 | handle_cast(send_heartbeat, State = #state{transport = T, socket = S}) -> 85 | spawn(fun() -> 86 | HBIOL = riak_repl2_rtframe:encode(heartbeat, undefined), 87 | T:send(S, HBIOL) 88 | end), 89 | {noreply, State}; 90 | 91 | handle_cast({v1_ack, Seq}, State = #state{v1_seq_map = Map}) -> 92 | case orddict:find(Seq, Map) of 93 | error -> 94 | ok; 95 | {ok, RealSeq} -> 96 | riak_repl2_rtq:ack(State#state.remote, RealSeq) 97 | end, 98 | Map2 = orddict:erase(Seq, Map), 99 | {noreply, State#state{v1_seq_map = Map2}}; 100 | 101 | handle_cast(Msg, _State) -> 102 | ?LOG_INFO("Realtime source helper received unexpected cast - ~p\n", [Msg]). 103 | 104 | 105 | handle_info(Msg, State) -> 106 | ?LOG_INFO("Realtime source helper received unexpected message - ~p\n", [Msg]), 107 | {noreply, State}. 108 | 109 | terminate(_Reason, _State) -> 110 | ok. 111 | 112 | code_change(_OldVsn, State, _Extra) -> 113 | {ok, State}. 114 | 115 | 116 | %% Trigger an async pull from the realtime queue 117 | async_pull(#state{remote = Remote, deliver_fun = Deliver}) -> 118 | riak_repl2_rtq:pull(Remote, Deliver). 119 | 120 | maybe_send(Transport, Socket, QEntry, State) -> 121 | {_Seq, _NumObjects, _BinObjs, Meta} = QEntry, 122 | #state{remote = Remote} = State, 123 | Routed = get_routed(Meta), 124 | case lists:member(Remote, Routed) of 125 | true -> 126 | ?LOG_DEBUG("Did not forward to ~p; destination already in routed list", [Remote]), 127 | State; 128 | false -> 129 | case State#state.proto of 130 | {Major, _Minor} when Major >= 3 -> 131 | encode_and_send(QEntry, Remote, Transport, Socket, State); 132 | _ -> 133 | case riak_repl_bucket_type_util:is_bucket_typed(Meta) of 134 | false -> 135 | encode_and_send(QEntry, Remote, Transport, Socket, State); 136 | true -> 137 | ?LOG_DEBUG("Negotiated protocol version:~p does not support typed buckets, not sending"), 138 | State 139 | end 140 | end 141 | end. 142 | 143 | encode_and_send(QEntry, Remote, Transport, Socket, State) -> 144 | QEntry2 = merge_forwards_and_routed_meta(QEntry, Remote), 145 | {Encoded, State2} = encode(QEntry2, State), 146 | ?LOG_DEBUG("Forwarding to ~p with new data: ~p derived from ~p", [State#state.remote, QEntry2, QEntry]), 147 | Transport:send(Socket, Encoded), 148 | State2. 149 | 150 | 151 | encode({Seq, _NumObjs, BinObjs, Meta}, State = #state{proto = Ver}) when Ver < {2,0} -> 152 | Skips = orddict:fetch(skip_count, Meta), 153 | Offset = State#state.v1_offset + Skips, 154 | Seq2 = Seq - Offset, 155 | V1Map = orddict:store(Seq2, Seq, State#state.v1_seq_map), 156 | BinObjs2 = riak_repl_util:maybe_downconvert_binary_objs(BinObjs, w0), 157 | Encoded = riak_repl2_rtframe:encode(objects, {Seq2, BinObjs2}), 158 | State2 = State#state{v1_offset = Offset, v1_seq_map = V1Map}, 159 | {Encoded, State2}; 160 | encode({Seq, _NumbOjbs, BinObjs, Meta}, State = #state{proto = Ver}) when Ver >= {2,0} -> 161 | {riak_repl2_rtframe:encode(objects_and_meta, {Seq, BinObjs, Meta}), State}. 162 | 163 | get_routed(Meta) -> 164 | meta_get(routed_clusters, [], Meta). 165 | 166 | meta_get(Key, Default, Meta) -> 167 | case orddict:find(Key, Meta) of 168 | error -> Default; 169 | {ok, Value} -> Value 170 | end. 171 | 172 | merge_forwards_and_routed_meta({_, _, _, Meta} = QEntry, Remote) -> 173 | LocalForwards = meta_get(local_forwards, [Remote], Meta), 174 | Routed = meta_get(routed_clusters, [], Meta), 175 | Self = riak_core_connection:symbolic_clustername(), 176 | Meta2 = orddict:erase(local_forwards, Meta), 177 | Routed2 = lists:usort(Routed ++ LocalForwards ++ [Self]), 178 | Meta3 = orddict:store(routed_clusters, Routed2, Meta2), 179 | setelement(4, QEntry, Meta3). 180 | -------------------------------------------------------------------------------- /src/riak_repl2_rtsource_sup.erl: -------------------------------------------------------------------------------- 1 | %% Riak EnterpriseDS 2 | %% Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved. 3 | -module(riak_repl2_rtsource_sup). 4 | -author('Andy Gross '). 5 | -behaviour(supervisor). 6 | 7 | %% External exports 8 | -export([start_link/0]). 9 | %% supervisor callbacks 10 | -export([init/1]). 11 | 12 | %% @spec start_link() -> ServerRet 13 | %% @doc API for starting the supervisor. 14 | start_link() -> 15 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 16 | 17 | %% @spec init([]) -> SupervisorTree 18 | %% @doc supervisor callback. 19 | init([]) -> 20 | Processes = 21 | [{riak_repl2_rtq, {riak_repl2_rtq, start_link, []}, 22 | transient, 50000, worker, [riak_repl2_rtq]}, 23 | 24 | {riak_repl2_rtq_overload_counter, {riak_repl2_rtq_overload_counter, start_link, []}, 25 | permanent, 50000, worker, [riak_repl2_rtq_overload_counter]}, 26 | 27 | {riak_repl2_rtsource_conn_sup, {riak_repl2_rtsource_conn_sup, start_link, []}, 28 | permanent, infinity, supervisor, [riak_repl2_rtsource_conn_sup]}], 29 | 30 | {ok, {{rest_for_one, 9, 10}, Processes}}. 31 | -------------------------------------------------------------------------------- /src/riak_repl_bq.erl: -------------------------------------------------------------------------------- 1 | -module(riak_repl_bq). 2 | -behaviour(gen_server). 3 | 4 | -export([start_link/2, q_ack/2, status/1, stop/1]). 5 | 6 | %% gen_server callbacks 7 | -export([init/1, handle_call/3, handle_cast/2, handle_info/2, 8 | terminate/2, code_change/3]). 9 | 10 | -include("riak_repl.hrl"). 11 | 12 | -record(state, { 13 | transport, 14 | socket, 15 | q, 16 | pending, 17 | max_pending, 18 | client 19 | }). 20 | 21 | start_link(Transport, Socket) -> 22 | gen_server:start_link(?MODULE, [Transport, Socket], []). 23 | 24 | q_ack(Pid, Count) -> 25 | gen_server:cast(Pid, {q_ack, Count}). 26 | 27 | status(Pid) -> 28 | try 29 | gen_server:call(Pid, status, infinity) 30 | catch 31 | _:_ -> 32 | [{queue, too_busy}] 33 | end. 34 | 35 | stop(Pid) -> 36 | gen_server:call(Pid, stop, infinity). 37 | 38 | %% gen_server 39 | 40 | init([Transport, Socket]) -> 41 | QSize = app_helper:get_env(riak_repl,queue_size, 42 | ?REPL_DEFAULT_QUEUE_SIZE), 43 | MaxPending = app_helper:get_env(riak_repl,server_max_pending, 44 | ?REPL_DEFAULT_MAX_PENDING), 45 | {ok, C} = riak:local_client(), 46 | {ok, #state{q = bounded_queue:new(QSize), 47 | max_pending = MaxPending, 48 | pending = 0, 49 | socket=Socket, 50 | transport=Transport, 51 | client=C 52 | }}. 53 | 54 | 55 | handle_cast({q_ack, Count}, State = #state{pending=Pending}) -> 56 | drain(State#state{pending=Pending - Count}). 57 | 58 | handle_call(status, _From, State = #state{q=Q}) -> 59 | Status = [{queue_pid, self()}, 60 | {dropped_count, bounded_queue:dropped_count(Q)}, 61 | {queue_length, bounded_queue:len(Q)}, 62 | {queue_byte_size, bounded_queue:byte_size(Q)}, 63 | {queue_max_size, bounded_queue:max_size(Q)}, 64 | {queue_percentage, (bounded_queue:byte_size(Q) * 100) div 65 | bounded_queue:max_size(Q)}, 66 | {queue_pending, State#state.pending}, 67 | {queue_max_pending, State#state.max_pending} 68 | ], 69 | {reply, Status, State}; 70 | handle_call(stop, _From, State) -> 71 | {stop, normal, ok, State}. 72 | 73 | handle_info({repl, RObj}, State) -> 74 | case riak_repl_util:repl_helper_send_realtime(RObj, State#state.client) of 75 | [] -> 76 | %% no additional objects to queue 77 | drain(enqueue(term_to_binary({diff_obj, RObj}), State)); 78 | Objects when is_list(Objects) -> 79 | %% enqueue all the objects the hook asked us to send as a list. 80 | %% They're enqueued together so that they can't be dumped from the 81 | %% queue piecemeal if it overflows 82 | NewState = enqueue([term_to_binary({diff_obj, O}) || 83 | O <- Objects ++ [RObj]], State), 84 | drain(NewState); 85 | cancel -> 86 | {noreply, State} 87 | end. 88 | 89 | code_change(_OldVsn, State, _Extra) -> 90 | {ok, State}. 91 | 92 | terminate(_Reason, _State) -> 93 | ok. 94 | 95 | %% internal 96 | 97 | drain(State=#state{q=Q,pending=P,max_pending=M}) when P < M -> 98 | case bounded_queue:out(Q) of 99 | {{value, Msg}, NewQ} -> 100 | drain(send_diffobj(Msg, State#state{q=NewQ})); 101 | {empty, NewQ} -> 102 | {noreply, State#state{q=NewQ}} 103 | end; 104 | drain(State) -> 105 | {noreply, State}. 106 | 107 | enqueue(Msg, State=#state{q=Q}) -> 108 | State#state{q=bounded_queue:in(Q,Msg)}. 109 | 110 | send_diffobj(Msgs, State0) when is_list(Msgs) -> 111 | %% send all the messages in the list 112 | %% we correctly increment pending, so we should get enough q_acks 113 | %% to restore pending to be less than max_pending when we're done. 114 | lists:foldl(fun(Msg, State) -> 115 | send_diffobj(Msg, State) 116 | end, State0, Msgs); 117 | send_diffobj(Msg,State=#state{transport=Transport,socket=Socket,pending=Pending}) -> 118 | _ = riak_repl_tcp_server:send(Transport, Socket, Msg), 119 | State#state{pending=Pending+1}. 120 | 121 | -------------------------------------------------------------------------------- /src/riak_repl_bucket_type_util.erl: -------------------------------------------------------------------------------- 1 | %% Riak EnterpriseDS 2 | %% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved. 3 | -module(riak_repl_bucket_type_util). 4 | 5 | %% @doc Utility functions for interacting with bucket types 6 | 7 | -export([bucket_props_match/1, 8 | bucket_props_match/2, 9 | is_bucket_typed/1, 10 | prop_get/3, 11 | property_hash/1]). 12 | 13 | -include("riak_repl.hrl"). 14 | 15 | -define(DEFAULT_BUCKET_TYPE, <<"default">>). 16 | 17 | -spec bucket_props_match(proplists:proplist()) -> boolean(). 18 | bucket_props_match(Props) -> 19 | case is_bucket_typed(Props) of 20 | true -> 21 | Type = prop_get(?BT_META_TYPE, ?DEFAULT_BUCKET_TYPE, Props), 22 | property_hash(Type) =:= prop_get(?BT_META_PROPS_HASH, undefined, Props); 23 | false -> 24 | %% This is not a typed bucket. Check if the remote 25 | %% side is also untyped. 26 | undefined =:= prop_get(?BT_META_PROPS_HASH, undefined, Props) 27 | end. 28 | 29 | -spec bucket_props_match(binary(), integer()) -> boolean(). 30 | bucket_props_match(Type, RemoteBucketTypeHash) -> 31 | property_hash(Type) =:= RemoteBucketTypeHash. 32 | 33 | -spec is_bucket_typed({error, no_type} | proplists:proplist()) -> boolean(). 34 | is_bucket_typed({error, no_type}) -> 35 | false; 36 | is_bucket_typed(Props) -> 37 | prop_get(?BT_META_TYPED_BUCKET, false, Props). 38 | 39 | -spec prop_get(atom() | binary(), term(), {error, no_type} | proplists:proplist()) -> term(). 40 | prop_get(_Key, Default, {error, no_type}) -> 41 | Default; 42 | prop_get(Key, Default, Props) -> 43 | case lists:keyfind(Key, 1, Props) of 44 | {Key, Value} -> 45 | Value; 46 | false -> 47 | Default 48 | end. 49 | 50 | -spec property_hash(binary()) -> undefined | integer(). 51 | property_hash(undefined) -> 52 | undefined; 53 | property_hash(Type) -> 54 | Defaults = riak_core_capability:get( 55 | {riak_repl, default_bucket_props_hash}, 56 | [consistent, datatype, n_val, allow_mult, last_write_wins]), 57 | riak_core_bucket_type:property_hash(Type, Defaults). 58 | -------------------------------------------------------------------------------- /src/riak_repl_cinfo.erl: -------------------------------------------------------------------------------- 1 | %% Riak EnterpriseDS 2 | %% Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved. 3 | 4 | -module(riak_repl_cinfo). 5 | 6 | -export([cluster_info_init/0, cluster_info_generator_funs/0]). 7 | 8 | %% @spec () -> term() 9 | %% @doc Required callback function for cluster_info: initialization. 10 | %% 11 | %% This function doesn't have to do anything. 12 | 13 | cluster_info_init() -> 14 | ok. 15 | 16 | %% @spec () -> list({string(), fun()}) 17 | %% @doc Required callback function for cluster_info: return list of 18 | %% {NameForReport, FunOfArity_1} tuples to generate ASCII/UTF-8 19 | %% formatted reports. 20 | 21 | cluster_info_generator_funs() -> 22 | [ 23 | {"Riak Repl status", fun status/1} 24 | ]. 25 | 26 | status(CPid) -> % CPid is the data collector's pid. 27 | cluster_info:format(CPid, "~p\n", [riak_repl_console:status(quiet)]). 28 | -------------------------------------------------------------------------------- /src/riak_repl_client_sup.erl: -------------------------------------------------------------------------------- 1 | %% Riak EnterpriseDS 2 | %% Copyright 2007-2009 Basho Technologies, Inc. All Rights Reserved. 3 | -module(riak_repl_client_sup). 4 | -author('Andy Gross '). 5 | -behaviour(supervisor). 6 | -include("riak_repl.hrl"). 7 | 8 | -include_lib("kernel/include/logger.hrl"). 9 | 10 | -export([start_link/0, init/1, stop/1]). 11 | -export([start_site/1, stop_site/1, running_site_procs/0, 12 | running_site_procs_rpc/0, ensure_sites/1]). 13 | 14 | start_site(SiteName) -> 15 | ?LOG_INFO("Starting replication site ~p", [SiteName]), 16 | ChildSpec = make_site(SiteName), 17 | supervisor:start_child(?MODULE, ChildSpec). 18 | 19 | stop_site(SiteName) -> 20 | ?LOG_INFO("Stopping replication site ~p", [SiteName]), 21 | _ = supervisor:terminate_child(?MODULE, SiteName), 22 | _ = supervisor:delete_child(?MODULE, SiteName). 23 | 24 | running_site_procs() -> 25 | [{SiteName, Pid} || {SiteName, Pid, _, _} <- supervisor:which_children(?MODULE)]. 26 | 27 | %% return the node along with the running sites for accounting 28 | running_site_procs_rpc() -> 29 | {node(), catch(running_site_procs())}. 30 | 31 | ensure_sites(Ring) -> 32 | ReplConfig = 33 | case riak_repl_ring:get_repl_config(Ring) of 34 | undefined -> 35 | riak_repl_ring:initial_config(); 36 | RC -> RC 37 | end, 38 | CurrentConfig = running_site_procs(), 39 | CurrentSites = lists:map(fun({SiteName, _Pid}) -> SiteName end, 40 | CurrentConfig), 41 | ConfiguredSites = [Site#repl_site.name || Site <- dict:fetch(sites, ReplConfig)], 42 | ToStop = sets:to_list( 43 | sets:subtract( 44 | sets:from_list(CurrentSites), 45 | sets:from_list(ConfiguredSites))), 46 | ToStart = sets:to_list( 47 | sets:subtract( 48 | sets:from_list(ConfiguredSites), 49 | sets:from_list(CurrentSites))), 50 | _ = [start_site(SiteName) || SiteName <- ToStart], 51 | _ = [stop_site(SiteName) || SiteName <- ToStop], 52 | ok. 53 | 54 | start_link() -> 55 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 56 | 57 | stop(_S) -> ok. 58 | 59 | %% @private 60 | init([]) -> 61 | {ok, {{one_for_one, 100, 10}, []}}. 62 | 63 | make_site(SiteName) -> 64 | {SiteName, {riak_repl_tcp_client, start_link, [SiteName]}, 65 | permanent, brutal_kill, worker, [riak_repl_tcp_client]}. 66 | -------------------------------------------------------------------------------- /src/riak_repl_fsm_common.erl: -------------------------------------------------------------------------------- 1 | %% Riak EnterpriseDS 2 | %% Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved. 3 | -module(riak_repl_fsm_common). 4 | -author('Andy Gross 12 | Transport:setopts(Socket, ?FSM_SOCKOPTS), 13 | {ok, Client} = riak:local_client(), 14 | PI = riak_repl_util:make_peer_info(), 15 | Partitions = riak_repl_util:get_partitions(PI#peer_info.ring), 16 | [{client, Client}, 17 | {partitions, Partitions}, 18 | {my_pi, PI}]. 19 | 20 | work_dir(Transport, Socket, SiteName) -> 21 | {ok, WorkRoot} = application:get_env(riak_repl, work_dir), 22 | SiteDir = SiteName ++ "-" ++ riak_repl_util:format_socketaddrs(Socket, Transport), 23 | WorkDir = filename:join(WorkRoot, SiteDir), 24 | ok = filelib:ensure_dir(filename:join(WorkDir, "empty")), 25 | {ok, WorkDir}. 26 | -------------------------------------------------------------------------------- /src/riak_repl_fullsync_worker.erl: -------------------------------------------------------------------------------- 1 | -module(riak_repl_fullsync_worker). 2 | 3 | -behaviour(gen_server). 4 | 5 | -include_lib("riak_kv/include/riak_kv_vnode.hrl"). 6 | -include("riak_repl.hrl"). 7 | 8 | -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, 9 | code_change/3]). 10 | -export([start_link/1, do_put/3, do_binput/3, do_binputs/5, do_get/7, do_get/8]). 11 | 12 | -export([do_binputs_internal/4]). %% Used for unit/integration testing, not public interface 13 | 14 | -record(state, {}). 15 | 16 | start_link(_Args) -> 17 | gen_server:start_link(?MODULE, [], []). 18 | 19 | do_put(Pid, Obj, Pool) -> 20 | gen_server:cast(Pid, {put, Obj, Pool}). 21 | 22 | %% Put a single object, encoded as replication binary from wire format 23 | do_binput(Pid, BinObj, Pool) -> 24 | %% safe to cast as the pool size will add backpressure on the sink 25 | gen_server:cast(Pid, {bin_put, BinObj, Pool}). 26 | 27 | do_binputs(Pid, BinObjs, DoneFun, Pool, Ver) -> 28 | %% safe to cast as the pool size will add backpressure on the sink 29 | gen_server:cast(Pid, {puts, BinObjs, DoneFun, Pool, Ver}). 30 | 31 | do_get(Pid, Bucket, Key, Transport, Socket, Pool, Ver) -> 32 | gen_server:call(Pid, {get, Bucket, Key, Transport, Socket, Pool, Ver}, infinity). 33 | 34 | do_get(Pid, Bucket, Key, Transport, Socket, Pool, Partition, Ver) -> 35 | gen_server:call(Pid, {get, Bucket, Key, Transport, Socket, Pool, Partition, Ver}, 36 | infinity). 37 | 38 | 39 | init([]) -> 40 | {ok, #state{}}. 41 | 42 | handle_call({get, B, K, Transport, Socket, Pool, Ver}, From, State) -> 43 | %% unblock the caller 44 | gen_server:reply(From, ok), 45 | %% do the get and send it to the client 46 | {ok, Client} = riak:local_client(), 47 | case riak_client:get(B, K, 1, ?REPL_FSM_TIMEOUT, Client) of 48 | {ok, RObj} -> 49 | %% we don't actually have the vclock to compare, so just send the 50 | %% key and let the other side sort things out. 51 | case riak_repl_util:repl_helper_send(RObj, Client) of 52 | cancel -> 53 | ok; 54 | Objects when is_list(Objects) -> 55 | %% Cindy: Santa, why can we encode our own binary object? 56 | %% Santa: Because the send() function will convert our tuple 57 | %% to a binary 58 | _ = [riak_repl_tcp_server:send(Transport, Socket, 59 | riak_repl_util:encode_obj_msg( 60 | Ver,{fs_diff_obj,O})) 61 | || O <- Objects], 62 | _ = riak_repl_tcp_server:send(Transport, Socket, 63 | riak_repl_util:encode_obj_msg( 64 | Ver,{fs_diff_obj,RObj})), 65 | ok 66 | end, 67 | ok; 68 | {error, notfound} -> 69 | ok; 70 | _ -> 71 | ok 72 | end, 73 | %% unblock this worker for more work (or death) 74 | poolboy:checkin(Pool, self()), 75 | {noreply, State}; 76 | %% Handle a get() request by sending the named object via the tcp server back 77 | %% to the tcp client. 78 | handle_call({get, B, K, Transport, Socket, Pool, Partition, Ver}, From, State) -> 79 | %% unblock the caller 80 | gen_server:reply(From, ok), 81 | 82 | {ok, Ring} = riak_core_ring_manager:get_my_ring(), 83 | OwnerNode = riak_core_ring:index_owner(Ring, Partition), 84 | 85 | Preflist = [{Partition, OwnerNode}], 86 | 87 | ReqID = make_req_id(), 88 | 89 | Req = riak_kv_requests:new_get_request({B, K}, ReqID), 90 | %% Assuming this function is called from a FSM process 91 | %% so self() == FSM pid 92 | riak_core_vnode_master:command(Preflist, 93 | Req, 94 | {raw, ReqID, self()}, 95 | riak_kv_vnode_master), 96 | 97 | receive 98 | {ReqID, Reply} -> 99 | case Reply of 100 | {r, {ok, RObj}, _, ReqID} -> 101 | %% we don't actually have the vclock to compare, so just send the 102 | %% key and let the other side sort things out. 103 | {ok, Client} = riak:local_client(), 104 | case riak_repl_util:repl_helper_send(RObj, Client) of 105 | cancel -> 106 | ok; 107 | Objects when is_list(Objects) -> 108 | %% Cindy: Santa, why can we encode our own binary object? 109 | %% Santa: Because, Cindy, the send() function accepts 110 | %% either a binary or a term. 111 | _ = [riak_repl_tcp_server:send(Transport, Socket, 112 | riak_repl_util:encode_obj_msg( 113 | Ver,{fs_diff_obj,O})) 114 | || O <- Objects], 115 | _ = riak_repl_tcp_server:send(Transport, Socket, 116 | riak_repl_util:encode_obj_msg( 117 | Ver,{fs_diff_obj,RObj})), 118 | ok 119 | end, 120 | ok; 121 | {r, {error, notfound}, _, ReqID} -> 122 | ok; 123 | _ -> 124 | ok 125 | end 126 | after 127 | ?REPL_FSM_TIMEOUT -> 128 | ok 129 | end, 130 | %% unblock this worker for more work (or death) 131 | poolboy:checkin(Pool, self()), 132 | {noreply, State}; 133 | handle_call(_Event, _From, State) -> 134 | {reply, ok, State}. 135 | 136 | handle_cast({put, RObj, Pool}, State) -> 137 | %% do the put 138 | riak_repl_util:do_repl_put(RObj), 139 | %% unblock this worker for more work (or death) 140 | poolboy:checkin(Pool, self()), 141 | {noreply, State}; 142 | handle_cast({bin_put, BinObj, Pool}, State) -> 143 | RObj = riak_repl_util:decode_bin_obj(BinObj), 144 | riak_repl_util:do_repl_put(RObj), 145 | poolboy:checkin(Pool, self()), % resume work 146 | {noreply, State}; 147 | handle_cast({puts, BinObjs, DoneFun, Pool, Ver}, State) -> 148 | ?MODULE:do_binputs_internal(BinObjs, DoneFun, Pool, Ver), % so it can be mecked 149 | {noreply, State}; 150 | handle_cast(_Event, State) -> 151 | {noreply, State}. 152 | 153 | handle_info(stop, State) -> 154 | {stop, normal, State}; 155 | handle_info(_Info, State) -> 156 | {noreply, State}. 157 | 158 | terminate(_Reason, _State) -> 159 | ok. 160 | 161 | code_change(_OldVsn, State, _Extra) -> 162 | {ok, State}. 163 | 164 | 165 | %% Put a list of objects b2d, reinsert back in the pool and call DoneFun. 166 | %% TODO: rename external 'do_blah' functions. rest of riak uses do_blah 167 | %% for internal work 168 | do_binputs_internal(BinObjs, DoneFun, Pool, Ver) -> 169 | %% TODO: add mechanism for detecting put failure so 170 | %% we can drop rtsink and have it resent 171 | Objects = riak_repl_util:from_wire(Ver, BinObjs), 172 | _ = [riak_repl_util:do_repl_put(Obj) || Obj <- Objects], 173 | poolboy:checkin(Pool, self()), 174 | %% let the caller know 175 | DoneFun(). 176 | 177 | make_req_id() -> 178 | erlang:phash2({self(), os:timestamp()}). % stolen from riak_client 179 | -------------------------------------------------------------------------------- /src/riak_repl_leader_helper.erl: -------------------------------------------------------------------------------- 1 | %% Riak EnterpriseDS 2 | %% Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved. 3 | 4 | %%=================================================================== 5 | %% 6 | %% Helper module for riak_repl_leader_helper used for holding 7 | %% elections for the replication leader. gen_leader is used 8 | %% for elections but must use a static list of leader candidates. 9 | %% To accomodate the dynamic nature of riak replication, the 10 | %% registered name for the module is based on a hash of the configured 11 | %% nodes in the system. This guarantees that the election will have 12 | %% the same candidates/workers. The main riak_repl_leader module restart 13 | %% this helper when the replication configuration changes. 14 | %% 15 | %% The approach of naming the gen_leader process after the candidates 16 | %% runs the risk of exhausting all available atoms on the node if there 17 | %% are enough changes to the nodes in the ring. In normal production 18 | %% use this is unlikely and the extra atoms will go away when the node 19 | %% is restarted. It was the best option of a bad bunch (try and 20 | %% create some kind of proxy process, change gen_leader to add the 21 | %% candidate hash to each message). 22 | %%=================================================================== 23 | -module(riak_repl_leader_helper). 24 | -behaviour(gen_leader). 25 | -export([start_link/4, leader_node/1, leader_node/2, refresh_leader/1]). 26 | -export([init/1,elected/3,surrendered/3,handle_leader_call/4, 27 | handle_leader_cast/3, from_leader/3, handle_call/4, 28 | handle_cast/3, handle_DOWN/3, handle_info/2, terminate/2, 29 | code_change/4]). 30 | 31 | -include_lib("kernel/include/logger.hrl"). 32 | 33 | -define(LEADER_OPTS, [{vardir, VarDir}, {bcast_type, all}]). 34 | 35 | -record(state, {leader_mod, % module name of replication leader 36 | local_pid, % pid of local replication leader gen_server 37 | elected_node, % node last elected 38 | elected_pid}).% pid or replication leader on elected node 39 | 40 | %%%=================================================================== 41 | %%% API 42 | %%%=================================================================== 43 | 44 | start_link(LeaderModule, LeaderPid, Candidates, Workers) -> 45 | %% Make a unique name for this list of candidates. gen_leader protocol is designed 46 | %% to only handle static lists of candidate nodes. Make sure this is true by 47 | %% basing the name of the server on the list of candidates. 48 | Hash = integer_to_list(erlang:phash2({Candidates, Workers})), 49 | CandidateHashStr = Hash, 50 | LeaderModuleStr = atom_to_list(LeaderModule), 51 | RegStr = LeaderModuleStr ++ "_" ++ CandidateHashStr, 52 | RegName = list_to_atom(RegStr), 53 | 54 | %% Make sure there is a unique directory for this election 55 | {ok, DataRootDir} = application:get_env(riak_repl, data_root), 56 | VarDir = filename:join(DataRootDir, RegStr), 57 | ok = filelib:ensure_dir(filename:join(VarDir, ".empty")), 58 | 59 | LOpts = [{vardir, VarDir},{workers, Workers}, {bcast_type, all}], 60 | gen_leader:start_link(RegName, Candidates, LOpts, ?MODULE, [LeaderModule, LeaderPid], []). 61 | 62 | leader_node(HelperPid) -> 63 | leader_node(HelperPid, 5000). 64 | 65 | leader_node(HelperPid, Timeout) -> 66 | gen_leader:call(HelperPid, leader_node, Timeout). 67 | 68 | %% Asyncrhonously request a refresh of the leader... gen_leader can 69 | %% block until a candidate node comes up 70 | refresh_leader(HelperPid) -> 71 | gen_leader:cast(HelperPid, refresh_leader). 72 | 73 | %%%=================================================================== 74 | %%% gen_leader callbacks 75 | %%%=================================================================== 76 | 77 | init([LeaderModule, LeaderPid]) -> 78 | {ok, #state{leader_mod=LeaderModule, local_pid=LeaderPid}}. 79 | 80 | elected(State, _NewElection, _Node) -> 81 | NewState = State#state{elected_node=node(), elected_pid=State#state.local_pid}, 82 | tell_who_leader_is(NewState), 83 | {ok, {i_am_leader, node(), State#state.local_pid}, NewState}. 84 | 85 | surrendered(State, {i_am_leader, Leader, ElectedPid}, _NewElection) -> 86 | NewState = State#state{elected_node=Leader, elected_pid=ElectedPid}, 87 | tell_who_leader_is(NewState), 88 | {ok, NewState}. 89 | 90 | handle_leader_call(Msg, _From, State, _E) -> 91 | {stop, {badmsg, Msg}, badmsg, State}. 92 | 93 | handle_leader_cast(Msg, State, _Election) -> 94 | {stop, {badmsg, Msg}, State}. 95 | 96 | from_leader({i_am_leader, _Leader, _ElectedPid}, State, _NewElection) -> 97 | %% sent as a result of the 2nd element of the elected/3 return tuple - ignored 98 | {ok, State}. 99 | 100 | handle_call(leader_node, _From, State, E) -> 101 | {reply, gen_leader:leader_node(E), State}. 102 | 103 | handle_cast(refresh_leader, State, _E) -> 104 | tell_who_leader_is(State), 105 | {noreply, State}. 106 | 107 | handle_DOWN(Node, State, _Election) -> 108 | %% only seems to fire when non-leader candidate nodes go down. not useful for 109 | %% replication purposes. 110 | ?LOG_INFO("Replication candidate node ~p down", [Node]), 111 | {ok, State}. 112 | 113 | handle_info(_Info, State) -> 114 | {noreply, State}. 115 | 116 | terminate(_Reason, _State) -> 117 | ok. 118 | 119 | code_change(_OldVsn, State, _Election, _Extra) -> 120 | {ok, State}. 121 | 122 | %%%=================================================================== 123 | %%% Internal functions 124 | %%%=================================================================== 125 | 126 | %% Tell who the main riak_repl_leader process who the leader is 127 | tell_who_leader_is(State) -> 128 | LeaderModule = State#state.leader_mod, 129 | LeaderModule:set_leader(State#state.local_pid, 130 | State#state.elected_node, 131 | State#state.elected_pid). 132 | 133 | 134 | -------------------------------------------------------------------------------- /src/riak_repl_listener_sup.erl: -------------------------------------------------------------------------------- 1 | %% Riak EnterpriseDS 2 | %% Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved. 3 | -module(riak_repl_listener_sup). 4 | -author('Andy Gross '). 5 | -include("riak_repl.hrl"). 6 | 7 | -include_lib("kernel/include/logger.hrl"). 8 | 9 | -export([start_listener/1, ensure_listeners/1, close_all_connections/0, 10 | server_pids/0]). 11 | 12 | 13 | start_listener(Listener = #repl_listener{listen_addr={IP, Port}}) -> 14 | case riak_repl_util:valid_host_ip(IP) of 15 | true -> 16 | ?LOG_INFO("Starting replication listener on ~s:~p", 17 | [IP, Port]), 18 | {ok, RawAddress} = inet_parse:address(IP), 19 | ranch:start_listener(Listener, 20 | ranch_tcp, 21 | [{ip, RawAddress}, 22 | {port, Port}, 23 | {num_acceptors, 10}], 24 | riak_repl_tcp_server, 25 | []); 26 | _ -> 27 | ?LOG_ERROR("Cannot start replication listener " 28 | "on ~s:~p - invalid address.", 29 | [IP, Port]) 30 | end. 31 | 32 | ensure_listeners(Ring) -> 33 | ReplConfig = 34 | case riak_repl_ring:get_repl_config(Ring) of 35 | undefined -> 36 | riak_repl_ring:initial_config(); 37 | RC -> RC 38 | end, 39 | CurrentListeners = [L || 40 | {{_, L}, Pid, _Type, _Modules} <- supervisor:which_children(ranch_sup), 41 | is_pid(Pid), is_record(L, repl_listener)], 42 | ConfiguredListeners = [Listener || Listener <- dict:fetch(listeners, ReplConfig), 43 | Listener#repl_listener.nodename == node()], 44 | ToStop = sets:to_list( 45 | sets:subtract( 46 | sets:from_list(CurrentListeners), 47 | sets:from_list(ConfiguredListeners))), 48 | ToStart = sets:to_list( 49 | sets:subtract( 50 | sets:from_list(ConfiguredListeners), 51 | sets:from_list(CurrentListeners))), 52 | _ = [start_listener(Listener) || Listener <- ToStart], 53 | lists:foreach(fun(Listener) -> 54 | {IP, Port} = Listener#repl_listener.listen_addr, 55 | ?LOG_INFO("Stopping replication listener on ~s:~p", 56 | [IP, Port]), 57 | ranch:stop_listener(Listener) 58 | end, ToStop), 59 | ok. 60 | 61 | close_all_connections() -> 62 | [exit(P, kill) || P <- 63 | server_pids()]. 64 | 65 | server_pids() -> 66 | %%%%%%%% 67 | %% NOTE: 68 | %% This is needed because Ranch doesn't directly expose child PID's. 69 | %% However, digging into the Ranch supervision tree can cause problems in the 70 | %% future if Ranch is upgraded. Ideally, this code should be moved into 71 | %% Ranch. Please check for Ranch updates! 72 | %%%%%%%% 73 | [Pid2 || 74 | {{ranch_listener_sup, _}, Pid, _Type, _Modules} <- supervisor:which_children(ranch_sup), is_pid(Pid), 75 | {ranch_conns_sup,Pid1,_,_} <- supervisor:which_children(Pid), 76 | {_,Pid2,_,_} <- supervisor:which_children(Pid1)]. 77 | -------------------------------------------------------------------------------- /src/riak_repl_migration.erl: -------------------------------------------------------------------------------- 1 | %% Riak Replication Realtime Migration manager 2 | %% Copyright (c) 2012 Basho Technologies, Inc. All Rights Reserved. 3 | 4 | -module(riak_repl_migration). 5 | 6 | -behaviour(gen_server). 7 | 8 | -include_lib("kernel/include/logger.hrl"). 9 | 10 | %% API 11 | -export([start_link/0,migrate_queue/0, migrate_queue/1]). 12 | 13 | %% gen_server callbacks 14 | -export([init/1, handle_call/3, handle_cast/2, handle_info/2, 15 | terminate/2, code_change/3]). 16 | 17 | -define(SERVER, ?MODULE). 18 | 19 | -record(state, {elapsed_sleep, 20 | caller}). 21 | 22 | start_link() -> 23 | gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). 24 | 25 | migrate_queue() -> 26 | DefaultTimeout = app_helper:get_env(riak_repl, queue_migration_timeout, 5), 27 | gen_server:call(?SERVER, {wait_for_queue, DefaultTimeout}, infinity). 28 | migrate_queue(Timeout) -> 29 | %% numeric timeout only... probably need to support infinity 30 | gen_server:call(?SERVER, {wait_for_queue, Timeout}, infinity). 31 | 32 | init([]) -> 33 | ?LOG_INFO("Riak replication migration server started"), 34 | {ok, #state{elapsed_sleep=0}}. 35 | 36 | handle_call({wait_for_queue, MaxTimeout}, From, State) -> 37 | ?LOG_INFO("Realtime Repl queue migration sleeping"), 38 | %% TODO: is there a better way to do the next line? just call 39 | %% handle_info? 40 | erlang:send_after(100, self(), {sleep, MaxTimeout}), 41 | {noreply, State#state{caller = From, elapsed_sleep = 0}}. 42 | 43 | handle_cast(_Msg, State) -> 44 | {noreply, State}. 45 | 46 | handle_info({sleep, MaxTimeout}, State = #state{elapsed_sleep = ElapsedSleep}) -> 47 | case riak_repl2_rtq:all_queues_empty() of 48 | true -> 49 | gen_server:reply(State#state.caller, ok), 50 | ?LOG_INFO("Queue empty, no replication queue migration required"); 51 | false -> 52 | case (ElapsedSleep >= MaxTimeout) of 53 | true -> 54 | ?LOG_INFO("Realtime queue has not completely drained"), 55 | _ = case riak_repl_util:get_peer_repl_nodes() of 56 | [] -> 57 | ?LOG_ERROR("No nodes available to migrate replication data"), 58 | riak_repl_stats:rt_source_errors(), 59 | error; 60 | [Peer|_Rest] -> 61 | {ok, _} = riak_repl2_rtq:register(qm), 62 | WireVer = riak_repl_util:peer_wire_format(Peer), 63 | ?LOG_INFO("Migrating replication queue data to ~p with wire version ~p", 64 | [Peer, WireVer]), 65 | drain_queue(riak_repl2_rtq:is_empty(qm), Peer, WireVer), 66 | ?LOG_INFO("Done migrating replication queue"), 67 | ok 68 | end, 69 | gen_server:reply(State#state.caller, ok); 70 | false -> 71 | ?LOG_INFO("Waiting for realtime repl queue to drain"), 72 | erlang:send_after(1000, self(), {sleep, MaxTimeout}) 73 | end 74 | end, 75 | NewState = State#state{elapsed_sleep = ElapsedSleep + 1000}, 76 | {noreply, NewState}. 77 | 78 | %% Drain the realtime queue and push all objects into a Peer node's input RT queue. 79 | %% If the handoff node does not understand the new repl wire format, then we need 80 | %% to downconvert the items into the old "w0" format, otherwise the other node will 81 | %% send an unsupported object format to its eventual sink node. This is painful to 82 | %% trace back to here. 83 | drain_queue(false, Peer, PeerWireVer) -> 84 | % would have made this a standard function, but I need a closure for the 85 | % value Peer 86 | riak_repl2_rtq:pull_sync(qm, 87 | fun ({Seq, NumItem, W1BinObjs, Meta}) -> 88 | try 89 | BinObjs = riak_repl_util:maybe_downconvert_binary_objs(W1BinObjs, PeerWireVer), 90 | CastObj = case PeerWireVer of 91 | w0 -> 92 | {push, NumItem, BinObjs}; 93 | w1 -> 94 | {push, NumItem, BinObjs, Meta} 95 | end, 96 | gen_server:cast({riak_repl2_rtq,Peer}, CastObj), 97 | %% Note - the next line is casting, not calling. 98 | riak_repl2_rtq:ack(qm, Seq) 99 | catch 100 | _:_ -> 101 | % probably too much spam in the logs for this warning 102 | %?LOG_WARNING("Dropped object during replication queue migration"), 103 | % is this the correct stat? 104 | riak_repl_stats:objects_dropped_no_clients(), 105 | riak_repl_stats:rt_source_errors() 106 | end, 107 | ok end), 108 | drain_queue(riak_repl2_rtq:is_empty(qm), Peer, PeerWireVer); 109 | 110 | drain_queue(true, _Peer, _Ver) -> 111 | done. 112 | 113 | terminate(_Reason, _State) -> 114 | ok. 115 | 116 | code_change(_OldVsn, State, _Extra) -> 117 | {ok, State}. 118 | 119 | %%%=================================================================== 120 | %%% Internal functions 121 | %%%=================================================================== 122 | -------------------------------------------------------------------------------- /src/riak_repl_ring_handler.erl: -------------------------------------------------------------------------------- 1 | %% Riak EnterpriseDS 2 | %% Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved. 3 | -module(riak_repl_ring_handler). 4 | -author('Andy Gross '). 5 | 6 | -behaviour(gen_event). 7 | 8 | -include("riak_repl.hrl"). 9 | 10 | -include_lib("kernel/include/logger.hrl"). 11 | 12 | %% gen_event callbacks 13 | -export([init/1, handle_event/2, handle_call/2, 14 | handle_info/2, terminate/2, code_change/3]). 15 | 16 | -export([update_leader/1]). 17 | 18 | -record(state, { ring :: tuple() }). 19 | 20 | 21 | %% =================================================================== 22 | %% gen_event Callbacks 23 | %% =================================================================== 24 | 25 | init([]) -> 26 | %% Give the leader the intial set of candidates 27 | {ok, Ring} = riak_core_ring_manager:get_my_ring(), 28 | AllNodes = riak_core_ring:all_members(Ring), 29 | riak_repl2_leader:set_candidates(AllNodes, []), 30 | _ = rt_update_events(Ring), 31 | _ = pg_update_events(Ring), 32 | {ok, #state{ring=Ring}}. 33 | 34 | handle_event({ring_update, Ring}, State=#state{ring=Ring}) -> 35 | %% Ring is unchanged from previous notification 36 | {ok, State}; 37 | handle_event({ring_update, NewRing}, State=#state{ring=OldRing}) -> 38 | %% Ring has changed. 39 | case riak_core_ring:check_lastgasp(NewRing) of 40 | true -> 41 | {ok, State}; 42 | false -> 43 | FinalRing = init_repl_config(OldRing, NewRing), 44 | update_leader(FinalRing), 45 | _ = rt_update_events(FinalRing), 46 | _ = pg_update_events(FinalRing), 47 | riak_repl_listener_sup:ensure_listeners(FinalRing), 48 | case riak_repl_leader:is_leader() of 49 | true -> 50 | riak_repl_leader:ensure_sites(); 51 | _ -> 52 | ok 53 | end, 54 | %% Force the cluster manager to connect to the clusters when it 55 | %% learns about an event *after* an election has occurred. 56 | case riak_repl2_leader:is_leader() of 57 | true -> 58 | riak_core_cluster_mgr:connect_to_clusters(); 59 | _ -> 60 | ok 61 | end, 62 | {ok, State#state{ring=FinalRing}} 63 | end; 64 | handle_event(_Event, State) -> 65 | {ok, State}. 66 | 67 | 68 | handle_call(_Request, State) -> 69 | {ok, ok, State}. 70 | 71 | 72 | handle_info(_Info, State) -> 73 | {ok, State}. 74 | 75 | 76 | terminate(_Reason, _State) -> 77 | ok. 78 | 79 | 80 | code_change(_OldVsn, State, _Extra) -> 81 | {ok, State}. 82 | 83 | 84 | %% =================================================================== 85 | %% Internal functions 86 | %% =================================================================== 87 | 88 | %% 89 | %% Initialize repl config structure as necessary 90 | %% 91 | init_repl_config(OldRing, NewRing) -> 92 | %% Check the repl config for changes. The ring is updated 93 | %% such that we are guaranteed to have an initialized data 94 | %% structure after this function returns. If necessary, we 95 | %% also push out the changed ring. 96 | OldRC = riak_repl_ring:get_repl_config(OldRing), 97 | NewRC = riak_repl_ring:get_repl_config(NewRing), 98 | case {OldRC, NewRC} of 99 | {undefined, undefined} -> 100 | update_ring(riak_repl_ring:initial_config()); 101 | {_, undefined} -> 102 | update_ring(OldRC); 103 | _ -> 104 | NewRing 105 | end. 106 | 107 | 108 | %% 109 | %% Given a new repl config, update the system-local ring. 110 | %% 111 | update_ring(ReplConfig) -> 112 | ?LOG_INFO("Repushing new REPL config!"), 113 | F = fun(Ring, _) -> 114 | {new_ring, riak_repl_ring:set_repl_config(Ring, ReplConfig)} 115 | end, 116 | {ok, FinalRing} = riak_core_ring_manager:ring_trans(F, undefined), 117 | FinalRing. 118 | 119 | 120 | %% 121 | %% Pass updated configuration settings to the leader 122 | %% 123 | update_leader(Ring) -> 124 | AllNodes = riak_core_ring:all_members(Ring), 125 | riak_repl2_leader:set_candidates(AllNodes, []), 126 | case riak_repl_ring:get_repl_config(Ring) of 127 | undefined -> 128 | ok; 129 | RC -> 130 | Listeners = listener_nodes(RC), 131 | NonListeners = ordsets:to_list( 132 | ordsets:subtract(ordsets:from_list(AllNodes), 133 | ordsets:from_list(Listeners))), 134 | 135 | case {has_sites(RC), has_listeners(RC)} of 136 | {_, true} -> 137 | Candidates=Listeners, 138 | Workers=NonListeners; 139 | {true, false} -> 140 | Candidates=AllNodes, 141 | Workers=[]; 142 | {false, false} -> 143 | Candidates=[], 144 | Workers=[] 145 | end, 146 | riak_repl_listener_sup:ensure_listeners(Ring), 147 | riak_repl_leader:set_candidates(Candidates, Workers) 148 | end. 149 | 150 | has_sites(ReplConfig) -> 151 | dict:fetch(sites, ReplConfig) /= []. 152 | 153 | has_listeners(ReplConfig) -> 154 | dict:fetch(listeners, ReplConfig) /= []. 155 | 156 | listener_nodes(ReplConfig) -> 157 | Listeners = dict:fetch(listeners, ReplConfig), 158 | lists:usort([L#repl_listener.nodename || L <- Listeners]). 159 | 160 | %% Run whenever the ring is changed or on startup. 161 | %% Compare desired state of realtime repl to configured 162 | rt_update_events(Ring) -> 163 | _ = riak_repl2_rt:ensure_rt(riak_repl_ring:rt_enabled(Ring), 164 | riak_repl_ring:rt_started(Ring)), 165 | %% ensure_rt sets this 166 | RTEnabled = app_helper:get_env(riak_repl, rtenabled, false), 167 | 168 | RC = case riak_repl_ring:get_repl_config(Ring) of 169 | undefined -> 170 | riak_repl_ring:initial_config(); 171 | R -> 172 | R 173 | end, 174 | 175 | LegacyEnabled = case {has_listeners(RC), has_sites(RC), 176 | app_helper:get_env(riak_repl, inverse_connection, false)} of 177 | {false, _, false} -> 178 | false; % No need to install hook if nobody is listening 179 | {true, _, false} -> 180 | true; 181 | {_, false, true} -> 182 | false; %% no sites 183 | {_, true, true} -> 184 | true 185 | end, 186 | 187 | case RTEnabled == LegacyEnabled of 188 | true -> 189 | ok; 190 | _ -> 191 | %% one of them must be true 192 | application:set_env(riak_repl, rtenabled, true) 193 | end, 194 | 195 | RTCascades = case dict:find(realtime_cascades, RC) of 196 | error -> 197 | always; 198 | {ok, RTCascadesFound} -> 199 | RTCascadesFound 200 | end, 201 | application:set_env(riak_repl, realtime_cascades, RTCascades), 202 | 203 | %% always 'install' the hook, the postcommit hooks will be toggled by 204 | %% the rtenabled environment variable 205 | riak_repl:install_hook(). 206 | 207 | pg_update_events(Ring) -> 208 | riak_repl2_pg:ensure_pg(riak_repl_ring:pg_enabled(Ring)). 209 | 210 | 211 | -------------------------------------------------------------------------------- /src/riak_repl_rtenqueue.erl: -------------------------------------------------------------------------------- 1 | %% -------------------------------------------------------------------------- 2 | %% 3 | %% riak_repl_rtenqueue - common client code for PB/HTTP to use for rtq enqueue 4 | %% 5 | %% 6 | %% This file is provided to you under the Apache License, 7 | %% Version 2.0 (the "License"); you may not use this file 8 | %% except in compliance with the License. You may obtain 9 | %% a copy of the License at 10 | %% 11 | %% http://www.apache.org/licenses/LICENSE-2.0 12 | %% 13 | %% Unless required by applicable law or agreed to in writing, 14 | %% software distributed under the License is distributed on an 15 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | %% KIND, either express or implied. See the License for the 17 | %% specific language governing permissions and limitations 18 | %% under the License. 19 | %% 20 | %% -------------------------------------------------------------------------- 21 | 22 | %% @doc riak local client style command for putting Riak bucket/key 23 | %% onto the realtime repl queue 24 | %% 25 | 26 | -module(riak_repl_rtenqueue). 27 | 28 | -export([rt_enqueue/4]). 29 | 30 | %% @doc Users of Riak sometimes want Riak to realtime replicate an 31 | %% object on demand (dropped messages, external divergence detection 32 | %% etc) This function gets the object stored under `Bucket' `Key', 33 | %% with the get options from `Options' and calls the postcommit 34 | %% function(s) defined for realtime IF realtime is enabled for 35 | %% `Bucket'. Returns `ok' if all went well, or `{error, Reason}' is 36 | %% there was a failure. Riak seems to support "hybrid" configurations 37 | %% with the possibility that two types of realtime queue are in use, 38 | %% therefore error may obscure partial success. 39 | -spec rt_enqueue(Bucket, Key, Options, Client) -> 40 | ok | Error 41 | when 42 | Bucket::riak_object:bucket(), 43 | Key::riak_object:key(), 44 | Options::riak_kv_get_fsm:options(), 45 | Client::riak_client:riak_client(), 46 | Error::{error, Reason}, 47 | Reason::term(). 48 | rt_enqueue(Bucket, Key, Options, Client) -> 49 | GetRes = riak_client:get(Bucket, Key, Options, Client), 50 | case GetRes of 51 | {ok, Object} -> 52 | rt_enqueue_object(Object); 53 | GetErr -> 54 | GetErr 55 | end. 56 | 57 | %% @private used by rt_equeue, once the object has been got. 58 | -spec rt_enqueue_object(riak_object:riak_object()) -> 59 | ok | {error, ErrReason::term()}. 60 | rt_enqueue_object(Object) -> 61 | BucketProps = get_bucket_props(Object), 62 | %% If repl is a thing, then get the postcommit hook(s) from the 63 | %% util (not the props) as the props does not name hooks, and may 64 | %% have many we don't care about. See riak_repl:fixup/2 65 | RTEnabled = app_helper:get_env(riak_repl, rtenabled, false), 66 | 67 | case proplists:get_value(repl, BucketProps) of 68 | Val when (Val==true orelse Val==realtime orelse Val==both), 69 | RTEnabled == true -> 70 | Hooks = riak_repl_util:get_hooks_for_modes(), 71 | run_rt_hooks(Object, Hooks, []); 72 | _ -> 73 | {error, realtime_not_enabled} 74 | end. 75 | 76 | 77 | -spec run_rt_hooks(riak_object:riak_object(), Hooks::list(), ResAcc::list()) -> 78 | ok | {error, Err::term()}. 79 | run_rt_hooks(_Object, _Hooks=[], Acc) -> 80 | lists:foldl(fun rt_repl_results/2, ok, Acc); 81 | run_rt_hooks(Object, [{struct, Hook} | Hooks], Acc) -> 82 | %% Bit jankey as it duplicates code in riak_kv_fsm postcommit 83 | ModBin = proplists:get_value(<<"mod">>, Hook), 84 | FunBin = proplists:get_value(<<"fun">>, Hook), 85 | 86 | Res = 87 | try 88 | Mod = binary_to_atom(ModBin, utf8), 89 | Fun = binary_to_atom(FunBin, utf8), 90 | Mod:Fun(Object) 91 | catch 92 | Class:Exception -> 93 | {error, {Hook, Class, Exception}} 94 | end, 95 | run_rt_hooks(Object, Hooks, [Res | Acc]). 96 | 97 | -spec rt_repl_results(Res, ResAcc) -> ResAcc when 98 | Res :: ok | Err, 99 | Err :: any(), 100 | ResAcc:: ok | {error, list(Err)}. 101 | rt_repl_results(_Res=ok, _ResAcc=ok) -> 102 | ok; 103 | rt_repl_results(ErrRes, _ResAcc=ok) -> 104 | {error, [ErrRes]}; 105 | rt_repl_results(ErrRes, {error, Errors}) -> 106 | {error, [ErrRes | Errors]}. 107 | 108 | %% @doc get the properties for a riak_kv_object's bucket. This code 109 | %% appears in a few places, and I was about to cut and paste it to yet 110 | %% another, and decided to give it home. 111 | -spec get_bucket_props(riak_object:riak_object()) -> list({atom(), any()}). 112 | get_bucket_props(RObj) -> 113 | Bucket = riak_object:bucket(RObj), 114 | {ok, DefaultProps} = application:get_env(riak_core, 115 | default_bucket_props), 116 | BucketProps = riak_core_bucket:get_bucket(Bucket), 117 | %% typed buckets never fall back to defaults 118 | case is_tuple(Bucket) of 119 | false -> 120 | lists:keymerge(1, lists:keysort(1, BucketProps), 121 | lists:keysort(1, DefaultProps)); 122 | true -> 123 | BucketProps 124 | end. 125 | -------------------------------------------------------------------------------- /src/riak_repl_server_sup.erl: -------------------------------------------------------------------------------- 1 | %% Riak EnterpriseDS 2 | %% Copyright 2007-2009 Basho Technologies, Inc. All Rights Reserved. 3 | -module(riak_repl_server_sup). 4 | -author('Andy Gross '). 5 | -behaviour(supervisor). 6 | -include("riak_repl.hrl"). 7 | -export([start_link/0, init/1, stop/1]). 8 | -export([start_server/1]). 9 | 10 | start_server(Sitename) -> 11 | supervisor:start_child(?MODULE, [Sitename]). 12 | 13 | start_link() -> 14 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 15 | 16 | stop(_S) -> ok. 17 | 18 | %% @private 19 | init([]) -> 20 | {ok, 21 | {{simple_one_for_one, 10, 10}, 22 | [{undefined, 23 | {riak_repl_tcp_server, start_link, []}, 24 | temporary, brutal_kill, worker, [riak_repl_tcp_server]}]}}. 25 | -------------------------------------------------------------------------------- /src/riak_repl_sup.erl: -------------------------------------------------------------------------------- 1 | %% Riak EnterpriseDS 2 | %% Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved. 3 | -module(riak_repl_sup). 4 | -author('Andy Gross '). 5 | -behaviour(supervisor). 6 | 7 | %% External exports 8 | -export([start_link/0]). 9 | %% supervisor callbacks 10 | -export([init/1]). 11 | 12 | %% @spec start_link() -> ServerRet 13 | %% @doc API for starting the supervisor. 14 | start_link() -> 15 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 16 | 17 | %% @spec init([]) -> SupervisorTree 18 | %% @doc supervisor callback. 19 | init([]) -> 20 | Processes = [ 21 | {riak_repl_client_sup, {riak_repl_client_sup, start_link, []}, 22 | permanent, infinity, supervisor, [riak_repl_client_sup]}, 23 | 24 | {riak_repl_server_sup, {riak_repl_server_sup, start_link, []}, 25 | permanent, infinity, supervisor, [riak_repl_server_sup]}, 26 | 27 | {riak_repl_leader, {riak_repl_leader, start_link, []}, 28 | permanent, 5000, worker, [riak_repl_leader]}, 29 | 30 | {riak_repl2_leader, {riak_repl2_leader, start_link, []}, 31 | permanent, 5000, worker, [riak_repl2_leader]}, 32 | 33 | {riak_core_cluster_mgr_sup, {riak_core_cluster_mgr_sup, start_link, []}, 34 | permanent, infinity, supervisor, [riak_cluster_mgr_sup]}, 35 | 36 | {riak_repl2_fs_node_reserver, {riak_repl2_fs_node_reserver, start_link, []}, 37 | permanent, infinity, worker, [riak_repl2_fs_node_reserver]}, 38 | 39 | {riak_repl2_rt_sup, {riak_repl2_rt_sup, start_link, []}, 40 | permanent, infinity, supervisor, [riak_repl2_rt_sup]}, 41 | 42 | {riak_repl2_fscoordinator_sup, {riak_repl2_fscoordinator_sup, start_link, []}, 43 | permanent, infinity, supervisor, [riak_repl2_fscoordinator_sup]}, 44 | 45 | {riak_repl2_fscoordinator_serv_sup, {riak_repl2_fscoordinator_serv_sup, start_link, []}, 46 | permanent, infinity, supervisor, [riak_repl2_fscoordinator_serv_sup]}, 47 | 48 | {riak_repl2_fssource_sup, {riak_repl2_fssource_sup, start_link, []}, 49 | permanent, infinity, supervisor, [riak_repl2_fssource_sup]}, 50 | 51 | {riak_repl2_fssink_pool, {riak_repl2_fssink_pool, start_link, []}, 52 | permanent, 5000, worker, [riak_repl2_fssink_pool, poolboy]}, 53 | 54 | {riak_repl2_fssink_sup, {riak_repl2_fssink_sup, start_link, []}, 55 | permanent, infinity, supervisor, [riak_repl2_fssink_sup]}, 56 | 57 | {riak_repl2_pg_proxy_sup, {riak_repl2_pg_proxy_sup, start_link, []}, 58 | permanent, infinity, supervisor, [riak_repl2_pg_proxy_sup]}, 59 | 60 | {riak_repl2_pg_sup, {riak_repl2_pg_sup, start_link, []}, 61 | permanent, infinity, supervisor, [riak_repl2_pg_sup]} 62 | 63 | ], 64 | 65 | {ok, {{one_for_one, 9, 10}, Processes}}. 66 | 67 | -------------------------------------------------------------------------------- /src/riak_repl_web.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riak_repl_web: setup Riak's REPL HTTP interface 4 | %% 5 | %% Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | -module(riak_repl_web). 23 | -export([dispatch_table/0]). 24 | 25 | dispatch_table() -> 26 | Props = props(), 27 | [ 28 | {["riak-repl", "stats"], riak_repl_wm_stats, []}, 29 | 30 | {["rtq", bucket_type, bucket, key], 31 | riak_repl_wm_rtenqueue, Props}, 32 | 33 | {["rtq", bucket, key], 34 | riak_repl_wm_rtenqueue, Props} 35 | ]. 36 | 37 | props() -> 38 | [{riak, local}, {bucket_type, <<"default">>}]. 39 | 40 | -------------------------------------------------------------------------------- /test/fixup_test.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% ------------------------------------------------------------------- 6 | 7 | -module(fixup_test). 8 | 9 | -include_lib("eunit/include/eunit.hrl"). 10 | -include("riak_repl.hrl"). 11 | 12 | -define(REPL_HOOK, [?REPL_HOOK12, ?REPL_HOOK_BNW]). 13 | 14 | fixup_test_() -> 15 | {foreach, 16 | fun() -> 17 | application:load(riak_core), 18 | application:set_env(riak_core, bucket_fixups, [{riak_repl, 19 | riak_repl}]), 20 | application:set_env(riak_repl, rtenabled, true), 21 | riak_core_bucket:append_bucket_defaults([{postcommit, []}]), 22 | RingEvtPid = maybe_start_link(riak_core_ring_events:start_link()), 23 | timer:sleep(10), %% next line counts on precesses being registered 24 | RingMgrPid = maybe_start_link(riak_core_ring_manager:start_link(test)), 25 | {RingEvtPid, RingMgrPid} 26 | 27 | end, 28 | fun({RingEvtPid, RingMgrPid}) -> 29 | stop_pid(RingMgrPid), 30 | stop_pid(RingEvtPid), 31 | application:unset_env(riak_core, bucket_fixups), 32 | application:unset_env(riak_core, default_bucket_props), 33 | application:unset_env(riak_repl, rtenabled) 34 | end, 35 | [ 36 | {"simple", fun simple/0}, 37 | {"pre-existing", fun preexisting_repl_hook/0}, 38 | {"other postcommit hook", fun other_postcommit_hook/0}, 39 | {"blank bucker", fun blank_bucket/0}, 40 | {"inherit from default bucket", fun inherit_from_default_bucket/0} 41 | ] 42 | }. 43 | 44 | maybe_start_link({ok, Pid}) -> 45 | Pid; 46 | maybe_start_link({error, {already_started, _}}) -> 47 | undefined. 48 | 49 | stop_pid(undefined) -> 50 | ok; 51 | stop_pid(Pid) -> 52 | unlink(Pid), 53 | exit(Pid, kill). 54 | 55 | simple() -> 56 | Props = riak_core_bucket:get_bucket("testbucket"), 57 | ?assertEqual([], 58 | proplists:get_value(postcommit, Props)), 59 | riak_core_bucket:set_bucket("testbucket", [{repl, true}]), 60 | Props2 = riak_core_bucket:get_bucket("testbucket"), 61 | ?assertEqual(?REPL_HOOK, 62 | proplists:get_value(postcommit, Props2)), 63 | riak_core_bucket:set_bucket("testbucket", [{repl, false}]), 64 | Props3 = riak_core_bucket:get_bucket("testbucket"), 65 | ?assertEqual([], 66 | proplists:get_value(postcommit, Props3)), 67 | ok. 68 | 69 | preexisting_repl_hook() -> 70 | riak_core_bucket:set_bucket("testbucket", [{postcommit, 71 | ?REPL_HOOK}]), 72 | Props = riak_core_bucket:get_bucket("testbucket"), 73 | ?assertEqual([], 74 | proplists:get_value(postcommit, Props)), 75 | riak_core_bucket:set_bucket("testbucket", [{repl, true}]), 76 | Props2 = riak_core_bucket:get_bucket("testbucket"), 77 | ?assertEqual(?REPL_HOOK, 78 | proplists:get_value(postcommit, Props2)), 79 | ok. 80 | 81 | other_postcommit_hook() -> 82 | riak_core_bucket:set_bucket("testbucket", [{postcommit, 83 | [my_postcommit_def()]}]), 84 | Props = riak_core_bucket:get_bucket("testbucket"), 85 | ?assertEqual([my_postcommit_def()], 86 | proplists:get_value(postcommit, Props)), 87 | riak_core_bucket:set_bucket("testbucket", [{repl, true}]), 88 | Props2 = riak_core_bucket:get_bucket("testbucket"), 89 | ?assertEqual([my_postcommit_def() | ?REPL_HOOK], 90 | proplists:get_value(postcommit, Props2)), 91 | riak_core_bucket:set_bucket("testbucket", [{repl, false}]), 92 | Props3= riak_core_bucket:get_bucket("testbucket"), 93 | ?assertEqual([my_postcommit_def()], 94 | proplists:get_value(postcommit, Props3)), 95 | ok. 96 | 97 | blank_bucket() -> 98 | application:set_env(riak_core, default_bucket_props, []), 99 | Props = riak_core_bucket:get_bucket("testbucket"), 100 | ?assertEqual(undefined, 101 | proplists:get_value(postcommit, Props)), 102 | riak_core_bucket:set_bucket("testbucket", [{repl, true}]), 103 | Props2 = riak_core_bucket:get_bucket("testbucket"), 104 | ?assertEqual(?REPL_HOOK, 105 | proplists:get_value(postcommit, Props2)), 106 | riak_core_bucket:set_bucket("testbucket", [{repl, false}]), 107 | Props3 = riak_core_bucket:get_bucket("testbucket"), 108 | ?assertEqual([], 109 | proplists:get_value(postcommit, Props3)), 110 | ok. 111 | 112 | inherit_from_default_bucket() -> 113 | riak_core_bucket:set_bucket("testbucket", [{foo, bar}]), 114 | Props = riak_core_bucket:get_bucket("testbucket"), 115 | ?assertEqual(undefined, 116 | proplists:get_value(repl, Props)), 117 | riak_kv_hooks:create_table(), 118 | riak_repl:install_hook(), 119 | Props2 = riak_core_bucket:get_bucket("testbucket"), 120 | ?assertEqual(true, 121 | proplists:get_value(repl, Props2)), 122 | ?assertEqual(?REPL_HOOK, 123 | proplists:get_value(postcommit, Props2)), 124 | riak_core_bucket:set_bucket("testbucket", [{repl, false}]), 125 | Props3 = riak_core_bucket:get_bucket("testbucket"), 126 | ?assertEqual(false, 127 | proplists:get_value(repl, Props3)), 128 | ?assertEqual([], 129 | proplists:get_value(postcommit, Props3)), 130 | Props4 = riak_core_bucket:get_bucket("noncustombucket"), 131 | ?assertEqual(true, 132 | proplists:get_value(repl, Props4)), 133 | ?assertEqual(?REPL_HOOK, 134 | proplists:get_value(postcommit, Props4)), 135 | ok. 136 | 137 | my_postcommit_def() -> 138 | {struct, [{<<"mod">>,atom_to_binary(?MODULE, latin1)}, 139 | {<<"fun">>,<<"postcommit">>}]}. 140 | -------------------------------------------------------------------------------- /test/riak_core_cluster_mgr_sup_tests.erl: -------------------------------------------------------------------------------- 1 | -module(riak_core_cluster_mgr_sup_tests). 2 | -compile([export_all, nowarn_export_all]). 3 | -ifdef(EQC). 4 | -include("riak_core_connection.hrl"). 5 | -include_lib("eunit/include/eunit.hrl"). 6 | 7 | murdering_test_() -> 8 | error_logger:tty(false), 9 | {spawn, 10 | [ 11 | {setup, fun() -> 12 | % if the cluster_mgr is restarted after the leader process is running, 13 | % it should ask about who the current leader is. 14 | Kids = [ 15 | {riak_repl_leader, {riak_repl_leader, start_link, []}, 16 | permanent, 5000, worker, [riak_repl_leader]}, 17 | {riak_repl2_leader, {riak_repl2_leader, start_link, []}, 18 | permanent, 5000, worker, [riak_repl2_leader]}, 19 | {riak_core_cluster_mgr_sup, {riak_core_cluster_mgr_sup, start_link, []}, 20 | permanent, infinity, supervisor, [riak_cluster_mgr_sup]} 21 | ], 22 | meck:new(riak_repl_sup, [passthrough]), 23 | meck:expect(riak_repl_sup, init, fun(_Args) -> 24 | {ok, {{one_for_one, 9, 10}, Kids}} 25 | end), 26 | meck:new(riak_core_node_watcher_events), 27 | meck:expect(riak_core_node_watcher_events, add_sup_callback, fun(_fn) -> 28 | ok 29 | end), 30 | meck:new(riak_core_node_watcher), 31 | meck:expect(riak_core_node_watcher, nodes, fun(_) -> 32 | [node()] 33 | end), 34 | application:start(ranch), 35 | application:set_env(riak_repl, data_root, "."), 36 | {ok, _Eventer} = riak_core_ring_events:start_link(), 37 | {ok, _RingMgr} = riak_core_ring_manager:start_link(test), 38 | {ok, _ClientSup} = riak_repl_client_sup:start_link(), 39 | {ok, TopSup} = riak_repl_sup:start_link(), 40 | riak_repl2_leader:register_notify_fun( 41 | fun riak_core_cluster_mgr:set_leader/2), 42 | riak_repl_leader:set_candidates([node()], []), 43 | riak_repl2_leader:set_candidates([node()], []), 44 | wait_for_leader(), 45 | TopSup 46 | end, 47 | fun(TopSup) -> 48 | process_flag(trap_exit, true), 49 | catch exit(TopSup, kill), 50 | catch(exit(whereis(riak_repl_client_sup), kill)), 51 | riak_core_ring_manager:stop(), 52 | catch exit(riak_core_ring_events, kill), 53 | application:stop(ranch), 54 | meck:unload(), 55 | process_flag(trap_exit, false), 56 | ok 57 | end, 58 | fun(_) -> [ 59 | 60 | {"Cluster Mgr knows of a leader", fun() -> 61 | ?assertNotEqual(undefined, riak_core_cluster_mgr:get_leader()) 62 | end}, 63 | 64 | {"Kill off the cluster manager, it can still find a leader", fun() -> 65 | OldPid = whereis(riak_core_cluster_manager), 66 | catch exit(OldPid, kill), 67 | WaitFun = fun() -> 68 | case whereis(riak_core_cluster_manager) of 69 | OP when OP == OldPid -> 70 | false; 71 | undefined -> 72 | false; 73 | _Pid -> 74 | Leader = riak_core_cluster_mgr:get_leader(), 75 | Leader =/= undefined 76 | end 77 | end, 78 | WaitRes = wait(WaitFun, 10, 400), 79 | ?assertEqual(ok, WaitRes) 80 | end} 81 | 82 | ] end}]}. 83 | 84 | pester(Fun) -> 85 | pester(Fun, 10, 100). 86 | 87 | pester(Fun, Wait, Times) -> 88 | pester(Fun, Wait, Times, []). 89 | 90 | pester(_Fun, _Wait, Times, Acc) when Times =< 0 -> 91 | lists:reverse(Acc); 92 | pester(Fun, Wait, Times, Acc) -> 93 | Res = Fun(), 94 | timer:sleep(Wait), 95 | pester(Fun, Wait, Times - 1, [Res | Acc]). 96 | 97 | wait_for_leader() -> 98 | WaitFun = fun() -> 99 | riak_repl2_leader:leader_node() =/= undefined 100 | end, 101 | wait(WaitFun). 102 | 103 | wait(WaitFun) -> 104 | wait(WaitFun, 10, 100). 105 | 106 | wait(_WaitFun, _Wait, Itor) when Itor =< 0 -> 107 | timeout; 108 | wait(WaitFun, Wait, Itor) -> 109 | case WaitFun() of 110 | true -> 111 | ok; 112 | _ -> 113 | timer:sleep(Wait), 114 | wait(WaitFun, Wait, Itor - 1) 115 | end. 116 | 117 | -endif. % EQC 118 | -------------------------------------------------------------------------------- /test/riak_core_connection_tests.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Eunit test cases for the Conn Manager Connection 4 | %% 5 | %% Copyright (c) 2013 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | 24 | -module(riak_core_connection_tests). 25 | -author("Chris Tilt"). 26 | -include_lib("eunit/include/eunit.hrl"). 27 | 28 | -include_lib("kernel/include/logger.hrl"). 29 | 30 | -export([test1service/5, connected/6, connect_failed/3]). 31 | 32 | -define(TEST_ADDR, { "127.0.0.1", 4097}). 33 | -define(MAX_CONS, 2). 34 | -define(TCP_OPTIONS, [{keepalive, true}, 35 | {nodelay, true}, 36 | {packet, 4}, 37 | {reuseaddr, true}, 38 | {active, false}]). 39 | 40 | %% host service functions 41 | test1service(_Socket, _Transport, {error, Reason}, Args, _Props) -> 42 | ?LOG_DEBUG("test1service failed with {error, ~p}", [Reason]), 43 | ?assert(Args == failed_host_args), 44 | ?assert(Reason == protocol_version_not_supported), 45 | {error, Reason}; 46 | test1service(_Socket, _Transport, {ok, {Proto, MyVer, RemoteVer}}, Args, Props) -> 47 | [ExpectedMyVer, ExpectedRemoteVer] = Args, 48 | RemoteClusterName = proplists:get_value(clustername, Props), 49 | ?LOG_DEBUG("test1service started with Args ~p Props ~p", [Args, Props]), 50 | ?assert(RemoteClusterName == "undefined"), 51 | ?assert(ExpectedMyVer == MyVer), 52 | ?assert(ExpectedRemoteVer == RemoteVer), 53 | ?assert(Proto == test1proto), 54 | timer:sleep(2000), 55 | {ok, self()}. 56 | 57 | %% client connection callbacks 58 | connected(_Socket, _Transport, {_IP, _Port}, {Proto, MyVer, RemoteVer}, Args, Props) -> 59 | [ExpectedMyVer, ExpectedRemoteVer] = Args, 60 | RemoteClusterName = proplists:get_value(clustername, Props), 61 | ?LOG_DEBUG("connected with Args ~p Props ~p", [Args, Props]), 62 | ?assert(RemoteClusterName == "undefined"), 63 | ?assert(Proto == test1proto), 64 | ?assert(ExpectedMyVer == MyVer), 65 | ?assert(ExpectedRemoteVer == RemoteVer), 66 | timer:sleep(2000). 67 | 68 | connect_failed({Proto,_Vers}, {error, Reason}, Args) -> 69 | ?LOG_DEBUG("connect_failed: Reason = ~p Args = ~p", [Reason, Args]), 70 | ?assert(Args == failed_client_args), 71 | ?assert(Reason == protocol_version_not_supported), 72 | ?assert(Proto == test1protoFailed). 73 | 74 | conection_test_() -> 75 | {spawn, 76 | [ 77 | {setup, 78 | fun() -> 79 | riak_core_ring_events:start_link(), 80 | riak_core_ring_manager:start_link(test), 81 | ok = application:start(ranch), 82 | {ok, _} = riak_core_service_mgr:start_link(?TEST_ADDR), 83 | error_logger:tty(false), 84 | ok 85 | end, 86 | fun(ok) -> 87 | process_flag(trap_exit, true), 88 | riak_core_service_mgr:stop(), 89 | riak_core_ring_manager:stop(), 90 | catch exit(riak_core_ring_events, kill), 91 | application:stop(ranch), 92 | process_flag(trap_exit, false), 93 | error_logger:tty(true), 94 | ok 95 | end, 96 | fun(_) -> [ 97 | {"started", ?_assert(is_pid(whereis(riak_core_service_manager)))}, 98 | 99 | {"set and get name", 100 | fun() -> 101 | riak_core_connection:set_symbolic_clustername("undefined"), 102 | Got = riak_core_connection:symbolic_clustername(), 103 | ?assertEqual("undefined", Got) 104 | end}, 105 | 106 | {"register service", 107 | fun() -> 108 | ExpectedRevs = [{1,0}, {1,1}], 109 | ServiceProto = {test1proto, [{2,1}, {1,0}]}, 110 | ServiceSpec = {ServiceProto, {?TCP_OPTIONS, ?MODULE, test1service, ExpectedRevs}}, 111 | riak_core_service_mgr:register_service(ServiceSpec, {round_robin,?MAX_CONS}), 112 | ?assert(riak_core_service_mgr:is_registered(test1proto)) 113 | end}, 114 | 115 | {"unregister service", 116 | fun() -> 117 | TestProtocolId = test1proto, 118 | riak_core_service_mgr:unregister_service(TestProtocolId), 119 | ?assertNot(riak_core_service_mgr:is_registered(TestProtocolId)) 120 | end}, 121 | 122 | {"protocol match", 123 | fun() -> 124 | ExpectedRevs = [{1,0}, {1,1}], 125 | ServiceProto = {test1proto, [{2,1}, {1,0}]}, 126 | ServiceSpec = {ServiceProto, {?TCP_OPTIONS, ?MODULE, test1service, ExpectedRevs}}, 127 | riak_core_service_mgr:register_service(ServiceSpec, {round_robin,?MAX_CONS}), 128 | 129 | % test protocol match 130 | ClientProtocol = {test1proto, [{0,1},{1,1}]}, 131 | ClientSpec = {ClientProtocol, {?TCP_OPTIONS, ?MODULE, [{1,1},{1,0}]}}, 132 | riak_core_connection:connect(?TEST_ADDR, ClientSpec), 133 | timer:sleep(1000) 134 | end}, 135 | 136 | {"failed protocol match", 137 | fun() -> 138 | %% start service 139 | SubProtocol = {{test1protoFailed, [{2,1}, {1,0}]}, 140 | {?TCP_OPTIONS, ?MODULE, test1service, failed_host_args}}, 141 | riak_core_service_mgr:register_service(SubProtocol, {round_robin,?MAX_CONS}), 142 | ?assert(riak_core_service_mgr:is_registered(test1protoFailed) == true), 143 | 144 | %% try to connect via a client that speaks 0.1 and 3.1. No Match with host! 145 | ClientProtocol = {test1protoFailed, [{0,1},{3,1}]}, 146 | ClientSpec = {ClientProtocol, {?TCP_OPTIONS, ?MODULE, failed_client_args}}, 147 | Got = riak_core_connection:sync_connect(?TEST_ADDR, ClientSpec), 148 | ?assertEqual({error, protocol_version_not_supported}, Got), 149 | 150 | timer:sleep(2000) 151 | end} 152 | ] end 153 | }]} . 154 | -------------------------------------------------------------------------------- /test/riak_core_service_mgr_tests.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Eunit test cases for the Service Manager 4 | %% 5 | %% Copyright (c) 2013 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | -module(riak_core_service_mgr_tests). 24 | -author("Chris Tilt"). 25 | -include_lib("eunit/include/eunit.hrl"). 26 | 27 | %%-define(TRACE(Stmt),Stmt). 28 | -define(TRACE(Stmt),ok). 29 | 30 | %% internal functions 31 | -export([testService/5, 32 | connected/6, connect_failed/3 33 | ]). 34 | 35 | %% my name and remote same because I need to talk to myself for testing 36 | -define(MY_CLUSTER_NAME, "bob"). 37 | -define(REMOTE_CLUSTER_NAME, "bob"). 38 | 39 | -define(REMOTE_CLUSTER_ADDR, {"127.0.0.1", 4096}). 40 | -define(TEST_ADDR, {"127.0.0.1", 4097}). 41 | -define(MAX_CONS, 2). 42 | -define(TCP_OPTIONS, [{keepalive, true}, 43 | {nodelay, true}, 44 | {packet, 4}, 45 | {reuseaddr, true}, 46 | {active, false}]). 47 | 48 | service_test_() -> 49 | {spawn, 50 | [ 51 | {setup, 52 | fun() -> 53 | riak_core_ring_events:start_link(), 54 | riak_core_ring_manager:start_link(test), 55 | ok = application:start(ranch), 56 | {ok, _Pid} = riak_core_service_mgr:start_link(?TEST_ADDR), 57 | ok 58 | end, 59 | fun(ok) -> 60 | process_flag(trap_exit, true), 61 | riak_core_ring_manager:stop(), 62 | catch exit(riak_core_ring_events, kill), 63 | application:stop(ranch), 64 | process_flag(trap_exit, false), 65 | ok 66 | end, 67 | fun(_) -> [ 68 | 69 | {"started", ?_assert(is_pid(whereis(riak_core_service_manager)))}, 70 | 71 | {"get services", ?_assertEqual([], gen_server:call(riak_core_service_manager, get_services))}, 72 | 73 | {"register service", fun() -> 74 | ExpectedRevs = [{1,0}, {1,0}], 75 | TestProtocol = {{testproto, [{1,0}]}, {?TCP_OPTIONS, ?MODULE, testService, ExpectedRevs}}, 76 | riak_core_service_mgr:register_service(TestProtocol, {round_robin,?MAX_CONS}), 77 | ?assert(riak_core_service_mgr:is_registered(testproto)) 78 | end}, 79 | 80 | {"unregister service", fun() -> 81 | TestProtocolId = testproto, 82 | riak_core_service_mgr:unregister_service(TestProtocolId), 83 | ?assertNot(riak_core_service_mgr:is_registered(testproto)) 84 | end}, 85 | 86 | {"register stats fun", fun() -> 87 | Self = self(), 88 | Fun = fun(Stats) -> 89 | Self ! Stats 90 | end, 91 | riak_core_service_mgr:register_stats_fun(Fun), 92 | GotStats = receive 93 | Term -> 94 | Term 95 | after 5500 -> 96 | timeout 97 | end, 98 | ?assertEqual([], GotStats) 99 | end}, 100 | 101 | {"start service test", fun() -> 102 | %% re-register the test protocol and confirm registered 103 | TestProtocol = {{testproto, [{1,0}]}, {?TCP_OPTIONS, ?MODULE, testService, [{1,0}, {1,0}]}}, 104 | riak_core_service_mgr:register_service(TestProtocol, {round_robin, ?MAX_CONS}), 105 | ?assert(riak_core_service_mgr:is_registered(testproto)), 106 | %register_service_test_d(), 107 | %% try to connect via a client that speaks our test protocol 108 | ExpectedRevs = {expectedToPass, [{1,0}, {1,0}]}, 109 | riak_core_connection:connect(?TEST_ADDR, {{testproto, [{1,0}]}, 110 | {?TCP_OPTIONS, ?MODULE, ExpectedRevs}}), 111 | %% allow client and server to connect and make assertions of success/failure 112 | timer:sleep(1000), 113 | Stats = riak_core_service_mgr:get_stats(), 114 | ?assertEqual([{testproto,0}], Stats) 115 | end}, 116 | 117 | {"pause existing services", fun() -> 118 | riak_core_service_mgr:stop(), 119 | %% there should be no services running now. 120 | %% now start a client and confirm failure to connect 121 | ExpectedArgs = expectedToFail, 122 | riak_core_connection:connect(?TEST_ADDR, {{testproto, [{1,0}]}, 123 | {?TCP_OPTIONS, ?MODULE, ExpectedArgs}}), 124 | %% allow client and server to connect and make assertions of success/failure 125 | timer:sleep(1000) 126 | end} 127 | 128 | ] end 129 | }]}. 130 | 131 | %%------------------------ 132 | %% Helper functions 133 | %%------------------------ 134 | 135 | %% Protocol Service functions 136 | testService(_Socket, _Transport, {error, _Reason}, _Args, _Props) -> 137 | ?assert(false); 138 | testService(_Socket, _Transport, {ok, {Proto, MyVer, RemoteVer}}, Args, _Props) -> 139 | ?TRACE(?debugMsg("testService started")), 140 | [ExpectedMyVer, ExpectedRemoteVer] = Args, 141 | ?assertEqual(ExpectedMyVer, MyVer), 142 | ?assertEqual(ExpectedRemoteVer, RemoteVer), 143 | ?assertEqual(Proto, testproto), 144 | %% timer:sleep(2000), 145 | {ok, self()}. 146 | 147 | %% Client side protocol callbacks 148 | connected(_Socket, _Transport, {_IP, _Port}, {Proto, MyVer, RemoteVer}, Args, _Props) -> 149 | ?TRACE(?debugMsg("testClient started")), 150 | {_TestType, [ExpectedMyVer, ExpectedRemoteVer]} = Args, 151 | ?assertEqual(Proto, testproto), 152 | ?assertEqual(ExpectedMyVer, MyVer), 153 | ?assertEqual(ExpectedRemoteVer, RemoteVer), 154 | timer:sleep(2000). 155 | 156 | connect_failed({_Proto,_Vers}, {error, Reason}, Args) -> 157 | case Args of 158 | expectedToFail -> 159 | ?assertEqual(Reason, econnrefused); 160 | {retry_test, _Stuff} -> 161 | ?TRACE(?debugFmt("connect_failed: during retry test: ~p", [Reason])), 162 | ok; 163 | _Other -> 164 | ?TRACE(?debugFmt("connect_failed: ~p with args = ~p", [Reason, _Other])), 165 | ?assert(false) 166 | end, 167 | timer:sleep(1000). 168 | -------------------------------------------------------------------------------- /test/riak_repl_cs_eqc.erl: -------------------------------------------------------------------------------- 1 | %% 2 | %% EQC test for CS replication, with just another impl 3 | %% 4 | 5 | -module(riak_repl_cs_eqc). 6 | -compile([export_all, nowarn_export_all]). 7 | 8 | -ifdef(EQC). 9 | -include_lib("eqc/include/eqc.hrl"). 10 | 11 | -ifdef(TEST). 12 | -include_lib("eunit/include/eunit.hrl"). 13 | -endif. 14 | 15 | -define(QC_OUT(P), 16 | eqc:on_output(fun(Str, Args) -> io:format(user, Str, Args) end, P)). 17 | 18 | 19 | decision_table() -> 20 | %% type, rt, fs 21 | [{buckets, ok, ok}, %% Configurable, but default 22 | {users, ok, ok}, %% Configurable, but default 23 | {blocks, cancel, ok}, 24 | {block_ts, cancel, cancel}, 25 | {manifests, ok, ok}, 26 | {gc, ok, ok}, %% riak-cs-gc 27 | {access, cancel, cancel}, 28 | {storage, cancel, cancel}, 29 | {mb, ok, ok}, %% Multibag 30 | {tss, cancel, cancel}, %% Tombstones in short 31 | {other, ok, ok}]. 32 | 33 | decision_table_v2() -> 34 | %% type, rt, fs 35 | [{buckets, ok, ok}, %% Configurable, but default 36 | {users, ok, ok}, %% Configurable, but default 37 | {blocks, cancel, ok}, 38 | {block_ts, ok, cancel}, 39 | {manifests, ok, ok}, 40 | {gc, ok, ok}, %% riak-cs-gc 41 | {access, cancel, cancel}, 42 | {storage, cancel, cancel}, 43 | {mb, ok, ok}, %% Multibag 44 | {tss, cancel, cancel}, %% Tombstones in short 45 | {other, ok, ok}]. 46 | 47 | decision_table_v2_blockrt() -> 48 | %% type, rt, fs 49 | [{buckets, ok, ok}, %% Configurable, but default 50 | {users, ok, ok}, %% Configurable, but default 51 | {blocks, ok, ok}, 52 | {block_ts, ok, cancel}, 53 | {manifests, ok, ok}, 54 | {gc, ok, ok}, %% riak-cs-gc 55 | {access, cancel, cancel}, 56 | {storage, cancel, cancel}, 57 | {mb, ok, ok}, %% Multibag 58 | {tss, cancel, cancel}, %% Tombstones in short 59 | {other, ok, ok}]. 60 | 61 | decision_table_v2_no_blockts_rt() -> 62 | %% type, rt, fs 63 | [{buckets, ok, ok}, %% Configurable, but default 64 | {users, ok, ok}, %% Configurable, but default 65 | {blocks, cancel, ok}, 66 | {block_ts, cancel, cancel}, 67 | {manifests, ok, ok}, 68 | {gc, ok, ok}, %% riak-cs-gc 69 | {access, cancel, cancel}, 70 | {storage, cancel, cancel}, 71 | {mb, ok, ok}, %% Multibag 72 | {tss, cancel, cancel}, %% Tombstones in short 73 | {other, ok, ok}]. 74 | 75 | bucket_name_to_type(<<"0b:", _/binary>>, true) -> block_ts; %% Block tombstones 76 | bucket_name_to_type(<<"0b:", _/binary>>, false) -> blocks; 77 | bucket_name_to_type(_, true) -> tss; %% Tombstones 78 | bucket_name_to_type(<<"moss.buckets">>, _) -> buckets; 79 | bucket_name_to_type(<<"moss.users">>, _) -> users; 80 | bucket_name_to_type(<<"0o:", _/binary>>, _) -> manifests; 81 | bucket_name_to_type(<<"riak-cs-gc">>, _) -> gc; 82 | bucket_name_to_type(<<"moss.access">>, _) -> access; 83 | bucket_name_to_type(<<"moss.storage">>, _) -> storage; 84 | bucket_name_to_type(<<"riak-cs-multibag">>, _) -> mb; 85 | bucket_name_to_type(Bin, _) when is_binary(Bin) -> other. 86 | 87 | 88 | do_send(RTorFS, Object, DecisionTable) -> 89 | Type = bucket_name_to_type(riak_object:bucket(Object), 90 | riak_kv_util:is_x_deleted(Object)), 91 | {Type, RT, FS} = lists:keyfind(Type, 1, DecisionTable), 92 | case RTorFS of 93 | rt -> RT; 94 | fs -> FS 95 | end. 96 | 97 | %% Bucket names in Riak, used by CS 98 | raw_cs_bucket() -> 99 | oneof([<<"moss.buckets">>, 100 | <<"moss.users">>, 101 | <<"0b:deadbeef">>, 102 | <<"0o:deadbeef">>, 103 | <<"riak-cs-gc">>, 104 | <<"moss.access">>, 105 | <<"moss.storage">>, 106 | <<"riak-cs-multibag">>, 107 | %% And this binary() is default fallback, although this is 108 | %% not supposed to happen in CS use cases. But just in 109 | %% case. 110 | binary()]). 111 | 112 | riak_object() -> 113 | ?LET({Bucket, Key, HasTombstone}, 114 | {raw_cs_bucket(), non_empty(binary()), bool()}, 115 | begin 116 | Object = riak_object:new(Bucket, Key, <<"val">>), 117 | case HasTombstone of 118 | true -> 119 | M = dict:from_list([{<<"X-Riak-Deleted">>, true}]), 120 | Object2 = riak_object:update_metadata(Object, M), 121 | riak_object:apply_updates(Object2); 122 | false -> 123 | Object 124 | end 125 | end). 126 | 127 | eqc_test_() -> 128 | NumTests = 1024, 129 | {inorder, 130 | [ % run qc tests 131 | ?_assertEqual(true, eqc:quickcheck(eqc:numtests(NumTests, ?QC_OUT(prop_main(v1))))), 132 | ?_assertEqual(true, eqc:quickcheck(eqc:numtests(NumTests, ?QC_OUT(prop_main(v2))))), 133 | ?_assertEqual(true, eqc:quickcheck(eqc:numtests(NumTests, ?QC_OUT(prop_main(v2_blockrt))))), 134 | ?_assertEqual(true, eqc:quickcheck(eqc:numtests(NumTests, ?QC_OUT(prop_main(v2_no_blockts_rt))))) 135 | ]}. 136 | 137 | fs_or_rt() -> oneof([fs, rt]). 138 | 139 | prop_main(DecisionTableVersion) -> 140 | ok = application:unset_env(riak_repl, replicate_cs_block_tombstone_realtime), 141 | ok = application:unset_env(riak_repl, replicate_cs_block_tombstone_fullsync), 142 | ok = application:unset_env(riak_repl, replicate_cs_blocks_realtime), 143 | DecisionTable 144 | = case DecisionTableVersion of 145 | v1 -> 146 | %% Default behaviour in Riak EE 1.4~2.1.1 147 | ok = application:set_env(riak_repl, 148 | replicate_cs_block_tombstone_realtime, 149 | false), 150 | decision_table(); 151 | v2 -> 152 | %% New behaviour since Riak EE 2.1.2? 153 | decision_table_v2(); 154 | v2_blockrt -> 155 | %% Replicate blocks in realtime, off by default 156 | ok = application:set_env(riak_repl, 157 | replicate_cs_blocks_realtime, 158 | true), 159 | decision_table_v2_blockrt(); 160 | v2_no_blockts_rt -> 161 | %% Prevent block tombstone propagation in realtime, on by default 162 | ok = application:set_env(riak_repl, 163 | replicate_cs_block_tombstone_realtime, 164 | false), 165 | decision_table_v2_no_blockts_rt() 166 | end, 167 | ?FORALL({Object, FSorRT}, {riak_object(), fs_or_rt()}, 168 | begin 169 | Impl = 170 | case FSorRT of 171 | fs -> riak_repl_cs:send(Object, fake_client); 172 | rt -> riak_repl_cs:send_realtime(Object, fake_client) 173 | end, 174 | Verify = do_send(FSorRT, Object, DecisionTable), 175 | %% ?debugVal({Verify, Impl, DecisionTableVersion}), 176 | Verify =:= Impl 177 | end). 178 | 179 | -endif. 180 | -------------------------------------------------------------------------------- /test/riak_repl_schema_tests.erl: -------------------------------------------------------------------------------- 1 | -module(riak_repl_schema_tests). 2 | 3 | -include_lib("eunit/include/eunit.hrl"). 4 | -compile([export_all, nowarn_export_all]). 5 | 6 | %% basic schema test will check to make sure that all defaults from 7 | %% the schema make it into the generated app.config 8 | basic_schema_test() -> 9 | %% The defaults are defined in priv/riak_repl.schema. 10 | %% it is the file under test. 11 | Config = cuttlefish_unit:generate_templated_config( 12 | ["priv/riak_repl.schema"], [], context()), 13 | io:format("~p~n", [Config]), 14 | 15 | cuttlefish_unit:assert_config(Config, "riak_repl.data_root", "./repl/root"), 16 | cuttlefish_unit:assert_config(Config, "riak_core.cluster_mgr", {"1.2.3.4", 1234}), 17 | cuttlefish_unit:assert_config(Config, "riak_repl.max_fssource_cluster", 5), 18 | cuttlefish_unit:assert_config(Config, "riak_repl.max_fssource_node", 1), 19 | cuttlefish_unit:assert_config(Config, "riak_repl.max_fssource_soft_retries", 100), 20 | cuttlefish_unit:assert_config(Config, "riak_repl.fssource_retry_wait", 60), 21 | cuttlefish_unit:assert_config(Config, "riak_repl.max_fssink_node", 1), 22 | cuttlefish_unit:assert_config(Config, "riak_repl.fullsync_on_connect", true), 23 | cuttlefish_unit:assert_not_configured(Config, "riak_repl.fullsync_interval"), 24 | cuttlefish_unit:assert_config(Config, "riak_repl.rtq_max_bytes", 104857600), 25 | cuttlefish_unit:assert_config(Config, "riak_repl.proxy_get", disabled), 26 | cuttlefish_unit:assert_config(Config, "riak_repl.rt_heartbeat_interval", 15), 27 | cuttlefish_unit:assert_config(Config, "riak_repl.rt_heartbeat_timeout", 15), 28 | ok. 29 | 30 | override_schema_test() -> 31 | %% Conf represents the riak.conf file that would be read in by cuttlefish. 32 | %% this proplists is what would be output by the conf_parse module 33 | Conf = [ 34 | {["mdc", "data_root"], "/some/repl/place"}, 35 | {["mdc", "cluster_manager"], {"4.3.2.1", 4321}}, 36 | {["mdc", "max_fssource_cluster"], 10}, 37 | {["mdc", "max_fssource_node"], 2}, 38 | {["mdc", "max_fssink_node"], 4}, 39 | {["mdc", "fullsync_on_connect"], false}, 40 | {["mdc", "fullsync_interval", "cluster1"], "15m"}, 41 | {["mdc", "fullsync_interval", "cluster2"], "1h"}, 42 | {["mdc", "rtq_max_bytes"], "50MB"}, 43 | {["mdc", "proxy_get"], on}, 44 | {["mdc", "realtime", "heartbeat_interval"], "15m"}, 45 | {["mdc", "realtime", "heartbeat_timeout"], "15d"}, 46 | {["mdc", "fssource_retry_wait"], "10m"}, 47 | {["mdc", "max_fssource_soft_retries"], 196} 48 | ], 49 | 50 | %% The defaults are defined in priv/riak_repl.schema. 51 | %% it is the file under test. 52 | Config = cuttlefish_unit:generate_templated_config( 53 | ["priv/riak_repl.schema"], Conf, context()), 54 | 55 | cuttlefish_unit:assert_config(Config, "riak_repl.data_root", "/some/repl/place"), 56 | cuttlefish_unit:assert_config(Config, "riak_core.cluster_mgr", {"4.3.2.1", 4321}), 57 | cuttlefish_unit:assert_config(Config, "riak_repl.max_fssource_cluster", 10), 58 | cuttlefish_unit:assert_config(Config, "riak_repl.max_fssource_node", 2), 59 | cuttlefish_unit:assert_config(Config, "riak_repl.max_fssink_node", 4), 60 | cuttlefish_unit:assert_config(Config, "riak_repl.fullsync_on_connect", false), 61 | cuttlefish_unit:assert_config(Config, "riak_repl.fullsync_interval.cluster1", 15), 62 | cuttlefish_unit:assert_config(Config, "riak_repl.fullsync_interval.cluster2", 60), 63 | cuttlefish_unit:assert_config(Config, "riak_repl.rtq_max_bytes", 52428800), 64 | cuttlefish_unit:assert_config(Config, "riak_repl.proxy_get", enabled), 65 | cuttlefish_unit:assert_config(Config, "riak_repl.rt_heartbeat_interval", 900), 66 | cuttlefish_unit:assert_config(Config, "riak_repl.rt_heartbeat_timeout", 1296000), 67 | cuttlefish_unit:assert_config(Config, "riak_repl.fssource_retry_wait", 600), 68 | cuttlefish_unit:assert_config(Config, "riak_repl.max_fssource_soft_retries", 196), 69 | ok. 70 | 71 | %% this context() represents the substitution variables that rebar 72 | %% will use during the build process. riak_jmx's schema file is 73 | %% written with some {{mustache_vars}} for substitution during 74 | %% packaging cuttlefish doesn't have a great time parsing those, so we 75 | %% perform the substitutions first, because that's how it would work 76 | %% in real life. 77 | context() -> 78 | [ 79 | {repl_data_root, "./repl/root"}, 80 | {cluster_manager_ip, "1.2.3.4"}, 81 | {cluster_manager_port, 1234} 82 | ]. 83 | -------------------------------------------------------------------------------- /test/riak_repl_sup_tests.erl: -------------------------------------------------------------------------------- 1 | -module(riak_repl_sup_tests). 2 | 3 | -ifdef(TEST). 4 | 5 | -include_lib("eunit/include/eunit.hrl"). 6 | 7 | % Why? Because it's nice to know that system can start. 8 | 9 | can_start_test_() -> 10 | {spawn, 11 | [ 12 | {setup, 13 | fun() -> 14 | error_logger:tty(false), 15 | %% core features that are needed 16 | {ok, _Eventer} = riak_core_ring_events:start_link(), 17 | {ok, _RingMgr} = riak_core_ring_manager:start_link(test), 18 | 19 | %% needed by the leaders 20 | meck:new(riak_core_node_watcher_events), 21 | meck:expect(riak_core_node_watcher_events, 22 | add_sup_callback, 23 | fun(_fn) -> 24 | ok 25 | end), 26 | meck:new(riak_core_node_watcher), 27 | meck:expect(riak_core_node_watcher, nodes, fun(_) -> [node()] end), 28 | 29 | %% needed by repl itself 30 | application:start(ranch), 31 | application:set_env(riak_repl, data_root, ".") 32 | end, 33 | fun(_) -> 34 | process_flag(trap_exit, true), 35 | riak_core_ring_manager:stop(), 36 | catch exit(riak_core_ring_events, kill), 37 | application:stop(ranch), 38 | meck:unload(riak_core_node_watcher), 39 | meck:unload(riak_core_node_watcher_events), 40 | process_flag(trap_exit, false), 41 | ok 42 | end, 43 | fun(_) -> 44 | [ 45 | fun() -> 46 | {ok, Pid} = riak_repl_sup:start_link(), 47 | timer:sleep(500), 48 | ?assert(is_process_alive(Pid)) 49 | end 50 | 51 | ] 52 | end 53 | }]}. 54 | 55 | -endif. 56 | -------------------------------------------------------------------------------- /test/riak_repl_test_util.erl: -------------------------------------------------------------------------------- 1 | -module(riak_repl_test_util). 2 | 3 | -ifdef(TEST). 4 | 5 | -include_lib("eunit/include/eunit.hrl"). 6 | 7 | -export([reset_meck/1, reset_meck/2]). 8 | -export([abstract_gen_tcp/0, abstract_stats/0, abstract_stateful/0]). 9 | -export([kill_and_wait/1, kill_and_wait/2]). 10 | -export([wait_until_down/1, wait_until_down/2]). 11 | -export([maybe_unload_mecks/1]). 12 | -export([start_test_ring/0, stop_test_ring/0]). 13 | 14 | start_test_ring() -> 15 | stop_test_ring(), 16 | riak_core_ring_events:start_link(), 17 | riak_core_ring_manager:start_link(test). 18 | 19 | stop_test_ring() -> 20 | kill_and_wait(riak_core_ring_events, kill), 21 | kill_and_wait(riak_core_ring_manager, kill). 22 | 23 | maybe_unload_mecks(Mecks) when is_list(Mecks) -> 24 | Unload = fun(Meck) -> 25 | try meck:unload(Meck) of 26 | ok -> ok 27 | catch 28 | error:{not_mocked, Meck} -> ok 29 | end 30 | end, 31 | [Unload(M) || M <- Mecks]. 32 | 33 | abstract_gen_tcp() -> 34 | reset_meck(gen_tcp, [unstick, passthrough, no_link, non_strict]), 35 | meck:expect(gen_tcp, setopts, fun(Socket, Opts) -> 36 | inet:setopts(Socket, Opts) 37 | end), 38 | meck:expect(gen_tcp, peername, fun(Socket) -> 39 | inet:peername(Socket) 40 | end), 41 | meck:expect(gen_tcp, sockname, fun(Socket) -> 42 | inet:sockname(Socket) 43 | end). 44 | 45 | abstract_stats() -> 46 | reset_meck(riak_repl_stats, [no_link]), 47 | meck:expect(riak_repl_stats, rt_source_errors, fun() -> ok end), 48 | meck:expect(riak_repl_stats, objects_sent, fun() -> ok end). 49 | 50 | abstract_stateful() -> 51 | reset_meck(stateful, [non_strict]), 52 | meck:expect(stateful, set, fun(Key, Val) -> 53 | Fun = fun() -> Val end, 54 | meck:expect(stateful, Key, Fun) 55 | end), 56 | meck:expect(stateful, delete, fun(Key) -> 57 | meck:delete(stateful, Key, 0) 58 | end). 59 | 60 | reset_meck(Mod) -> 61 | reset_meck(Mod, []). 62 | 63 | reset_meck(Mod, Opts) -> 64 | try meck:unload(Mod) of 65 | ok -> ok 66 | catch 67 | error:{not_mocked, Mod} -> ok 68 | end, 69 | meck:new(Mod, Opts). 70 | 71 | kill_and_wait(Victims) -> 72 | kill_and_wait(Victims, stupify). 73 | 74 | kill_and_wait(undefined, _Cause) -> 75 | ok; 76 | 77 | kill_and_wait([], _Cause) -> 78 | ok; 79 | kill_and_wait([Atom | Rest], Cause) -> 80 | kill_and_wait(Atom, Cause), 81 | kill_and_wait(Rest, Cause); 82 | 83 | kill_and_wait(Atom, Cause) when is_atom(Atom) -> 84 | kill_and_wait(whereis(Atom), Cause); 85 | 86 | kill_and_wait(Pid, Cause) when is_pid(Pid) -> 87 | unlink(Pid), 88 | exit(Pid, Cause), 89 | wait_until_down(Pid). 90 | 91 | wait_until_down(Pid) -> 92 | wait_until_down(Pid, infinity). 93 | 94 | wait_until_down(Pid, Timeout) -> 95 | Mon = erlang:monitor(process, Pid), 96 | receive 97 | {'DOWN', Mon, process, Pid, _Why} -> 98 | ok 99 | after Timeout -> 100 | {error, timeout} 101 | end. 102 | 103 | -endif. --------------------------------------------------------------------------------