├── .github └── workflows │ └── erlang.yml ├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── docs ├── balanced.png ├── claim-fixes.md ├── claim-version4.md ├── hashtree.md ├── hashtree.png ├── node_worker_pool.md ├── rack-awareness.md ├── ring-location.png ├── ring-resizing.md └── unbalanced.png ├── eqc ├── bg_manager_eqc.erl ├── bprops_eqc.erl ├── btypes_eqc.erl ├── chash_eqc.erl ├── core_vnode_eqc.erl ├── hashtree_eqc.erl ├── new_cluster_membership_model_eqc.erl ├── node_watcher_qc.erl ├── riak_core_ring_eqc.erl ├── vclock_qc.erl └── worker_pool_pulse.erl ├── include ├── riak_core.hrl ├── riak_core_bg_manager.hrl ├── riak_core_bucket_type.hrl ├── riak_core_dtrace.hrl ├── riak_core_handoff.hrl ├── riak_core_metadata.hrl ├── riak_core_ring.hrl └── riak_core_vnode.hrl ├── priv └── riak_core.schema ├── rebar.config ├── rebar3 ├── src ├── app_helper.erl ├── bloom.erl ├── chash.erl ├── chashbin.erl ├── dvvset.erl ├── gen_nb_server.erl ├── hashtree.erl ├── hashtree_tree.erl ├── mock_vnode.erl ├── process_proxy.erl ├── riak_core.app.src ├── riak_core.erl ├── riak_core_apl.erl ├── riak_core_app.erl ├── riak_core_bg_manager.erl ├── riak_core_broadcast.erl ├── riak_core_broadcast_handler.erl ├── riak_core_bucket.erl ├── riak_core_bucket_props.erl ├── riak_core_bucket_type.erl ├── riak_core_capability.erl ├── riak_core_cinfo_core.erl ├── riak_core_claim_binring_alg.erl ├── riak_core_claim_sim.erl ├── riak_core_claim_swapping.erl ├── riak_core_claim_util.erl ├── riak_core_claimant.erl ├── riak_core_cli_registry.erl ├── riak_core_cluster_cli.erl ├── riak_core_console.erl ├── riak_core_console_table.erl ├── riak_core_coverage_fsm.erl ├── riak_core_coverage_plan.erl ├── riak_core_dist_mon.erl ├── riak_core_dtrace.erl ├── riak_core_eventhandler_guard.erl ├── riak_core_eventhandler_sup.erl ├── riak_core_exo_monitor.erl ├── riak_core_format.erl ├── riak_core_gen_server.erl ├── riak_core_gossip.erl ├── riak_core_handoff_cli.erl ├── riak_core_handoff_listener.erl ├── riak_core_handoff_listener_sup.erl ├── riak_core_handoff_manager.erl ├── riak_core_handoff_receiver.erl ├── riak_core_handoff_receiver_sup.erl ├── riak_core_handoff_sender.erl ├── riak_core_handoff_sender_sup.erl ├── riak_core_handoff_status.erl ├── riak_core_handoff_sup.erl ├── riak_core_location.erl ├── riak_core_membership_claim.erl ├── riak_core_membership_leave.erl ├── riak_core_metadata.erl ├── riak_core_metadata_exchange_fsm.erl ├── riak_core_metadata_hashtree.erl ├── riak_core_metadata_manager.erl ├── riak_core_metadata_object.erl ├── riak_core_mochiglobal.erl ├── riak_core_net_ticktime.erl ├── riak_core_node_watcher.erl ├── riak_core_node_watcher_events.erl ├── riak_core_node_worker_pool.erl ├── riak_core_node_worker_pool_sup.erl ├── riak_core_nodeid.erl ├── riak_core_priority_queue.erl ├── riak_core_pw_auth.erl ├── riak_core_repair.erl ├── riak_core_ring.erl ├── riak_core_ring_events.erl ├── riak_core_ring_handler.erl ├── riak_core_ring_manager.erl ├── riak_core_ring_util.erl ├── riak_core_security.erl ├── riak_core_send_msg.erl ├── riak_core_ssl_util.erl ├── riak_core_stat.erl ├── riak_core_stat_cache.erl ├── riak_core_stat_calc_proc.erl ├── riak_core_stat_calc_sup.erl ├── riak_core_stat_q.erl ├── riak_core_stat_sup.erl ├── riak_core_stat_xform.erl ├── riak_core_stats_sup.erl ├── riak_core_status.erl ├── riak_core_sup.erl ├── riak_core_supps.erl ├── riak_core_sysmon_handler.erl ├── riak_core_sysmon_minder.erl ├── riak_core_table_owner.erl ├── riak_core_tcp_mon.erl ├── riak_core_test_util.erl ├── riak_core_throttle.erl ├── riak_core_tracer.erl ├── riak_core_util.erl ├── riak_core_vnode.erl ├── riak_core_vnode_manager.erl ├── riak_core_vnode_master.erl ├── riak_core_vnode_proxy.erl ├── riak_core_vnode_proxy_sup.erl ├── riak_core_vnode_sup.erl ├── riak_core_vnode_worker.erl ├── riak_core_vnode_worker_pool.erl ├── riak_core_worker_pool.erl └── vclock.erl └── test ├── 13node_12node_ring.eqc ├── 169_group_join.eqc ├── 648_unbalanced_singly.eqc ├── bg_manager_tests.erl ├── bucket_fixup_test.erl ├── claim-statem-leaving-nodes-still-claim.eqc ├── claim_32_5_unbalanced.eqc ├── claim_simulation.erl ├── my_ring ├── rack_awareness_test.erl ├── riak_core_claim_eqc.erl ├── riak_core_claim_statem.erl ├── riak_core_schema_tests.erl ├── riak_core_security_tests.erl ├── riak_core_throttle_tests.erl ├── site1-cert.pem ├── site1-key.pem ├── site2-cert.pem ├── site2-key.pem ├── sync_command_test.erl ├── test_guarded_event_handler.erl └── worker_pool_test.erl /.github/workflows/erlang.yml: -------------------------------------------------------------------------------- 1 | name: Erlang CI 2 | 3 | on: 4 | push: 5 | branches: [ develop ] 6 | pull_request: 7 | branches: [ develop ] 8 | 9 | 10 | jobs: 11 | 12 | build: 13 | 14 | name: Test on ${{ matrix.os }} with OTP ${{ matrix.otp }} 15 | runs-on: ${{ matrix.os }} 16 | 17 | strategy: 18 | fail-fast: false 19 | matrix: 20 | otp: [22, 24, 25] 21 | os: [ubuntu-latest] 22 | # OTP lower than 23 does not run on ubuntu-latest (22.04), see 23 | # https://github.com/erlef/setup-beam#compatibility-between-operating-system-and-erlangotp 24 | exclude: 25 | - otp: 22 26 | os: ubuntu-latest 27 | include: 28 | - otp: 22 29 | os: ubuntu-20.04 30 | 31 | steps: 32 | - uses: lukka/get-cmake@latest 33 | - uses: actions/checkout@v2 34 | - name: Install dependencies (Ubuntu) 35 | if: ${{ startsWith(matrix.os, 'ubuntu') }} 36 | run: | 37 | sudo apt-get -qq update 38 | sudo apt-get -qq install libsnappy-dev libc6-dev 39 | - name: Install Erlang/OTP 40 | uses: erlef/setup-beam@v1 41 | with: 42 | otp-version: ${{ matrix.otp }} 43 | - name: Compile 44 | run: ./rebar3 compile 45 | - name: Run xref and dialyzer 46 | run: ./rebar3 do xref, dialyzer 47 | - name: Run eunit 48 | run: ./rebar3 as gha do eunit 49 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .eunit/* 2 | deps/* 3 | priv/* 4 | ebin/* 5 | *.o 6 | include/*_pb.hrl 7 | *~ 8 | doc/* 9 | /.eqc-info 10 | /current_counterexample.eqc 11 | .local_dialyzer_plt 12 | dialyzer_warnings 13 | dialyzer_unhandled_warnings 14 | .rebar 15 | .rebar3 16 | _build 17 | rebar.lock 18 | .DS_Store 19 | nonode@nohost 20 | data/* 21 | test/*.beam 22 | log/* 23 | core_vnode_eqc.log 24 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: compile rel cover test dialyzer eqc 2 | REBAR=./rebar3 3 | 4 | compile: 5 | $(REBAR) compile 6 | 7 | clean: 8 | $(REBAR) clean 9 | 10 | cover: 11 | $(REBAR) eunit --cover 12 | $(REBAR) cover 13 | 14 | test: compile 15 | $(REBAR) eunit 16 | 17 | dialyzer: 18 | $(REBAR) dialyzer 19 | 20 | xref: 21 | $(REBAR) xref 22 | 23 | eqc: 24 | $(REBAR) as test eqc --testing_budget 120 25 | $(REBAR) as eqc eunit 26 | 27 | check: test dialyzer xref 28 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Riak Core 2 | 3 | [![Erlang CI Actions Status](https://github.com/basho/riak_core/workflows/Erlang%20CI/badge.svg)](https://github.com/basho/riak_core/actions) 4 | 5 | Riak Core is the distributed systems framework that forms the basis of 6 | how [Riak](http://github.com/basho/riak) distributes data and scales. 7 | More generally, it can be thought of as a toolkit for building 8 | distributed, scalable, fault-tolerant applications. 9 | 10 | For some introductory reading on Riak Core (that’s not pure code), 11 | there’s an old but still valuable 12 | [blog post on the Basho Blog](http://basho.com/where-to-start-with-riak-core/) 13 | that’s well worth your time. 14 | 15 | ## Contributing 16 | 17 | We love community code, bug fixes, and other forms of contribution. We 18 | use GitHub Issues and Pull Requests for contributions to this and all 19 | other code. To get started: 20 | 21 | 1. Fork this repository. 22 | 2. Clone your fork or add the remote if you already have a clone of 23 | the repository. 24 | 3. Create a topic branch for your change. 25 | 4. Make your change and commit. Use a clear and descriptive commit 26 | message, spanning multiple lines if detailed explanation is needed. 27 | 5. Push to your fork of the repository and then send a pull request. 28 | 29 | 6. A Riak committer will review your patch and merge it into the main 30 | repository or send you feedback. 31 | 32 | ## Issues, Questions, and Bugs 33 | 34 | There are numerous ways to file issues or start conversations around 35 | something Core related 36 | 37 | * The 38 | [Riak Users List](http://lists.basho.com/mailman/listinfo/riak-users_lists.basho.com) 39 | is the main place for all discussion around Riak. 40 | * There is a 41 | [Riak Core-specific mailing list](http://lists.basho.com/mailman/listinfo/riak-core_lists.basho.com) 42 | for issues and questions that pertain to Core but not Riak. 43 | * #riak on Freenode is a very active channel and can get you some 44 | real-time help if a lot of instances. 45 | * If you've found a bug in Riak Core, 46 | [file](https://github.com/basho/riak_core/issues) a clear, concise, 47 | explanatory issue against this repo. 48 | -------------------------------------------------------------------------------- /docs/balanced.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/basho/riak_core/25d9a6fa917eb8a2e95795d64eb88d7ad384ed88/docs/balanced.png -------------------------------------------------------------------------------- /docs/hashtree.md: -------------------------------------------------------------------------------- 1 | `hashtree.erl` implements a fixed-sized hash tree, avoiding any need 2 | for rebalancing. The tree consists of a fixed number of on-disk 3 | `segments` and a hash tree constructed over these `segments`. Each 4 | level of the tree is grouped into buckets based on a fixed `tree 5 | width`. Each hash at level `i` corresponds to the hash of a bucket of 6 | hashes at level `i+1`. The following figure depicts a tree with 16 7 | segments and a tree-width of 4: 8 | 9 | ![image](https://github.com/basho/riak_kv/raw/jdb-hashtree/docs/hashtree.png) 10 | 11 | To insert a new `(key, hash)` pair, the key is hashed and mapped to 12 | one of the segments. The `(key, hash)` pair is then stored in the 13 | appropriate segment, which is an ordered `(key, hash)` dictionary. The 14 | given segment is then marked as dirty. Whenever `update_tree` is 15 | called, the hash for each dirty segment is re-computed, the 16 | appropriate leaf node in the hash tree updated, and the hash tree is 17 | updated bottom-up as necessary. Only paths along which hashes have 18 | been changed are re-computed. 19 | 20 | The current implementation uses LevelDB for the heavy lifting. Rather 21 | than reading/writing the on-disk segments as a unit, `(key, hash)` 22 | pairs are written to LevelDB as simple key-value pairs. The LevelDB 23 | key written is the binary `<<$s, SegmentId:64/integer, 24 | Key/binary>>`. Thus, inserting a new key-value hash is nothing more 25 | than a single LevelDB write. Likewise, key-hash pairs for a segment 26 | are laided on sequentially on-disk based on key sorting. An in-memory 27 | bitvector is used to track dirty segments, although a `gb_sets` was 28 | formerly used. 29 | 30 | When updating the segment hashes, a LevelDB iterator is used to access 31 | the segment keys in-order. The iterator seeks to the beginning of the 32 | segment and then iterators through all of the key-hash pairs. As an 33 | optimization, the iteration process is designed to read in multiple 34 | segments when possible. For example, if the list of dirty segments was 35 | `[1, 2, 3, 5, 6, 10]`, the code will seek an iterator to the beginning 36 | of segment 1, iterator through all of its keys, compute the 37 | appropriate segment 1 hash, then continue to traverse through segment 38 | 2 and segment 3's keys, updating those hashes as well. After segment 39 | 3, a new iterator will be created to seek to the beginning of segment 40 | 5, and handle both 5, and 6; and then a final iterator used to access 41 | segment 10. This design works very well when constructing a new tree 42 | from scratch. There's a phase of inserting a bunch of key-hash pairs 43 | (all writes), followed by an in-order traversal of the LevelDB 44 | database (all reads). 45 | 46 | Trees are compared using standard hash tree approach, comparing the 47 | hash at each level, and recursing to the next level down when 48 | different. After reaching the leaf nodes, any differing hashes results 49 | in a key exchange of the keys in the associated differing segments. 50 | 51 | By default, the hash tree itself is entirely in-memory. However, the 52 | code provides a `MEM_LEVEL` paramemter that specifics that levels 53 | greater than the parameter should be stored on-disk instead. These 54 | buckets are simply stored on disk in the same LevelDB structure as 55 | `{$b, Level, Bucket} -> orddict(Key, Hash)}` objects. 56 | 57 | The default settings use `1024*1024` segments with a tree width of 58 | `1024`. Thus, the resulting tree is only 3 levels deep. And there 59 | are only `1+1024+1024*1024` hashs stored in memory -- so, a few 60 | MB per hash tree. Given `1024*1024` on-disk segments, and assuming 61 | the code uniformly hashes keys to each segment, you end up with ~1000 62 | keys per segment with a 1 billion key hash tree. Thus, a single key 63 | difference would require 3 hash exchanges and a key exchange of 64 | 1000 keys to determine the differing key. 65 | -------------------------------------------------------------------------------- /docs/hashtree.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/basho/riak_core/25d9a6fa917eb8a2e95795d64eb88d7ad384ed88/docs/hashtree.png -------------------------------------------------------------------------------- /docs/rack-awareness.md: -------------------------------------------------------------------------------- 1 | # Rack Awareness / Availability zones / Location support 2 | 3 | The aim is to be able to increase data safety, and make the cluster more resilient 4 | against a location/site/availability zone/rack loss. 5 | 6 | To achieve this, a location parameter has been introduced. 7 | It can be set at runtime for each RIAK node. 8 | When claiming a new ring, the list of nodes is ordered taking into consideration the 9 | location of the individual nodes, in a manner that adjacent nodes are preferably 10 | from different locations. 11 | 12 | Basically it only changes the order of the nodes fed into the claiming algorithm. 13 | 14 | The default location is `undefined`. This means every node with no location parameter set 15 | will be handled as being in the same location. 16 | 17 | ## Ring visualization 18 | 19 | ![RIAK Ring Location](ring-location.png) 20 | 21 | ## Setup node's location parameter 22 | 23 | Setting up nodes’ location parameter is a staged operation like 24 | other ring manipulations (join, leave, resize-ring, etc). 25 | 26 | ### via riak admin 27 | Change current node location parameter: 28 | ```bash 29 | riak admin cluster location rack_a 30 | ``` 31 | or specify a node: 32 | ```bash 33 | riak admin cluster location site_b --node=dev2@127.0.0.1 34 | ``` 35 | 36 | #### by erlang function call 37 | 38 | ```erlang 39 | riak_core_claimant:set_node_location(node(), "location_a"), 40 | ``` 41 | ```erlang 42 | riak_core_claimant:plan(), 43 | riak_core_claimant:comit(). 44 | ``` 45 | 46 | ## Pitfalls 47 | There are circumstances in which the preferable node location assignment cannot be guaranteed. 48 | 49 | If at least one location parameter is set in the cluster when planning a cluster change, a warning 50 | message will be displayed when not all nodes in a preflist are assigned to a different location. 51 | 52 | For example, if the default `n_val = 3` is specified and there are only `two distinct locations` set in the cluster, 53 | the message `WARNING: Not all replicas will be on distinct locations` will be shown. 54 | 55 | ### Not enough distinct locations 56 | When Distinct Location Count is not divisible by Ring size. 57 | 58 | ### Tail violations 59 | When Ring Size not divisible by Count Of Nodes. 60 | [claim-fixes](claim-fixes.md) cover this, but improper distinct location count could result in undesirable location distribution within the ring. 61 | 62 | For example, there are 8 nodes on 3 distinct locations. 63 | To ensure that every site/location has a piece of data, n_val must be at least 4. 64 | 65 | It can be checked: 66 | 67 | Stages changes: 68 | ```erlang 69 | PlannedRing = element(1, lists:last(element(3, riak_core_claimant:plan()))). 70 | riak_core_location:check_ring(PlannedRing, Nval = 4, MinimumNumberOfDistinctLocations = 3). 71 | ``` 72 | 73 | Actual ring: 74 | ```erlang 75 | {ok, Ring} = riak_core_ring_manager:get_my_ring(), 76 | riak_core_location:check_ring(Ring, Nval = 4, MinimumNumberOfDistinctLocations = 3). 77 | ``` 78 | 79 | If `riak_core_location:check_ring/3` returns with an empty list `[]`, there is no location violation. 80 | 81 | ### Won't optimize transfers between old and new ring 82 | When location parameter change triggers ring ownership change, it currently does not optimize transfers. 83 | -------------------------------------------------------------------------------- /docs/ring-location.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/basho/riak_core/25d9a6fa917eb8a2e95795d64eb88d7ad384ed88/docs/ring-location.png -------------------------------------------------------------------------------- /docs/unbalanced.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/basho/riak_core/25d9a6fa917eb8a2e95795d64eb88d7ad384ed88/docs/unbalanced.png -------------------------------------------------------------------------------- /eqc/chash_eqc.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% chash_eqc: QuickCheck tests for the chash module. 4 | %% 5 | %% Copyright (c) 2007-2011 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | %% @doc QuickCheck tests for the chash module 24 | 25 | -module(chash_eqc). 26 | 27 | -ifdef(EQC). 28 | -include_lib("eqc/include/eqc.hrl"). 29 | -include_lib("eunit/include/eunit.hrl"). 30 | 31 | -define(NOTEST, true). 32 | -define(NOASSERT, true). 33 | 34 | -define(TEST_ITERATIONS, 50). 35 | -define(QC_OUT(P), 36 | eqc:on_output(fun(Str, Args) -> io:format(user, Str, Args) end, P)). 37 | -define(RINGTOP, trunc(math:pow(2,160)-1)). % SHA-1 space 38 | 39 | -export([check/0, 40 | test/0, 41 | test/1]). 42 | 43 | %%==================================================================== 44 | %% eunit test 45 | %%==================================================================== 46 | 47 | eqc_test_() -> 48 | {inparallel, 49 | [{spawn, 50 | [{setup, 51 | fun setup/0, 52 | fun cleanup/1, 53 | [ 54 | %% Run the quickcheck tests 55 | {timeout, 60000, % timeout is in msec 56 | %% Indicate the number of test iterations for each property here 57 | ?_assertEqual(true, 58 | quickcheck(numtests(?TEST_ITERATIONS, 59 | ?QC_OUT(prop_chash_next_index())))) 60 | } 61 | ] 62 | } 63 | ] 64 | } 65 | ] 66 | }. 67 | 68 | setup() -> 69 | %% Remove the logger noise. 70 | application:load(sasl), 71 | error_logger:tty(false), 72 | %% Uncomment the following lines to send log output to files. 73 | %% application:set_env(sasl, sasl_error_logger, {file, "chash_eqc_sasl.log"}), 74 | %% error_logger:logfile({open, "chash_eqc.log"}), 75 | 76 | %% TODO: Perform any required setup 77 | ok. 78 | 79 | cleanup(_) -> 80 | %% TODO: Perform any required cleanup 81 | ok. 82 | 83 | %% ==================================================================== 84 | %% eqc property 85 | %% ==================================================================== 86 | prop_chash_next_index() -> 87 | ?FORALL( 88 | {PartitionExponent, Delta}, 89 | {g_partition_exponent(), int()}, 90 | ?TRAPEXIT( 91 | begin 92 | %% Calculate the number of paritions 93 | NumPartitions = trunc(math:pow(2, PartitionExponent)), 94 | %% Calculate the integer indexes around the ring 95 | %% for the number of partitions. 96 | Inc = ?RINGTOP div NumPartitions, 97 | Indexes = [Inc * X || X <- lists:seq(0, NumPartitions-1)], 98 | %% Create a chash tuple to use for calls to chash:successors/2 99 | %% and chash:next_index/2. 100 | %% The node value is not used and so just use the default 101 | %% localhost node value. 102 | Node = 'riak@127.0.0.1', 103 | CHash = {NumPartitions, [{Index, Node} || Index <- Indexes]}, 104 | %% For each index around the ring add Delta to 105 | %% the index value and collect the results from calling 106 | %% chash:successors/2 and chash:next_index/2 for comparison. 107 | Results = 108 | [{element( 109 | 1, 110 | hd(chash:successors(<<(((Index + Delta) + ?RINGTOP) 111 | rem ?RINGTOP):160/integer>>, 112 | CHash))), 113 | chash:next_index((((Index + Delta) + ?RINGTOP) rem ?RINGTOP), 114 | CHash)} || 115 | Index <- Indexes], 116 | {ExpectedIndexes, ActualIndexes} = lists:unzip(Results), 117 | ?WHENFAIL( 118 | begin 119 | io:format("ExpectedIndexes: ~p AcutalIndexes: ~p~n", 120 | [ExpectedIndexes, ActualIndexes]) 121 | end, 122 | conjunction( 123 | [ 124 | {results, equals(ExpectedIndexes, ActualIndexes)} 125 | ])) 126 | end 127 | )). 128 | 129 | %%==================================================================== 130 | %% Generators 131 | %%==================================================================== 132 | 133 | g_partition_exponent() -> 134 | choose(1, 12). 135 | 136 | %%==================================================================== 137 | %% Helpers 138 | %%==================================================================== 139 | 140 | test() -> 141 | test(100). 142 | 143 | test(N) -> 144 | quickcheck(numtests(N, prop_chash_next_index())). 145 | 146 | check() -> 147 | check(prop_chash_next_index(), current_counterexample()). 148 | 149 | -endif. % EQC 150 | -------------------------------------------------------------------------------- /eqc/riak_core_ring_eqc.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% 4 | %% Copyright (c) 2013 Basho Technologies, Inc. All Rights Reserved. 5 | %% 6 | %% This file is provided to you under the Apache License, 7 | %% Version 2.0 (the "License"); you may not use this file 8 | %% except in compliance with the License. You may obtain 9 | %% a copy of the License at 10 | %% 11 | %% http://www.apache.org/licenses/LICENSE-2.0 12 | %% 13 | %% Unless required by applicable law or agreed to in writing, 14 | %% software distributed under the License is distributed on an 15 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | %% KIND, either express or implied. See the License for the 17 | %% specific language governing permissions and limitations 18 | %% under the License. 19 | %% 20 | %% ------------------------------------------------------------------- 21 | 22 | -module(riak_core_ring_eqc). 23 | 24 | -ifdef(EQC). 25 | -export([prop_future_index/0]). 26 | 27 | -include_lib("eqc/include/eqc.hrl"). 28 | -include_lib("eunit/include/eunit.hrl"). 29 | 30 | -define(TEST_ITERATIONS, 10000). 31 | -define(QC_OUT(P), 32 | eqc:on_output(fun(Str, Args) -> io:format(user, Str, Args) end, P)). 33 | 34 | 35 | eqc_test_() -> 36 | {inparallel, 37 | [{spawn, 38 | [{setup, 39 | fun() -> ok end, 40 | fun(_) -> ok end, 41 | [ 42 | %% Run the quickcheck tests 43 | {timeout, 60000, % timeout is in msec 44 | %% Indicate the number of test iterations for each property here 45 | ?_assertEqual(true, 46 | quickcheck(numtests(?TEST_ITERATIONS, 47 | ?QC_OUT(prop_future_index())))) 48 | }]}]}]}. 49 | 50 | 51 | 52 | prop_future_index() -> 53 | ?FORALL({CHashKey, OrigIdx, TargetIdx, N, Pos, Ring}=TransferItem, resize_item(), 54 | ?WHENFAIL(prop_future_index_failed(TransferItem), 55 | collect(N =/= undefined andalso Pos >= N, 56 | begin 57 | {Time, Val} = timer:tc(riak_core_ring, future_index, 58 | [CHashKey, OrigIdx, N, Ring]), 59 | measure(future_index_usec, Time, TargetIdx =:= Val) 60 | end))). 61 | 62 | resize_item() -> 63 | %% RingSize - Starting Ring Size 64 | %% GrowthFactor - >1 expanding, <1 shrinking 65 | %% IndexStart - first partition in preflist for key 66 | %% Pos - position of source partition in preflist 67 | %% N - optional known N-value for the key 68 | ?LET({RingSize, GrowthF}, 69 | {current_size(), growth_factor()}, 70 | ?LET({IndexStart, Pos, N}, 71 | {index_in_current_ring(RingSize), 72 | replica_position(RingSize), 73 | check_nval(RingSize, GrowthF)}, 74 | begin 75 | Ring0 = riak_core_ring:fresh(RingSize, node()), 76 | Ring1 = riak_core_ring:resize(Ring0,trunc(GrowthF * RingSize)), 77 | Ring = riak_core_ring:set_pending_resize(Ring1, Ring0), 78 | CHashKey = <<(IndexStart-1):160/integer>>, 79 | Preflist = riak_core_ring:preflist(CHashKey, Ring0), 80 | FuturePreflist = riak_core_ring:preflist(CHashKey, Ring1), 81 | {SourceIdx, _} = lists:nth(Pos+1, Preflist), 82 | 83 | %% account for case where position is greater than 84 | %% future ring size or possbly known N-value 85 | %% (shrinking) we shouldn't have a target index in 86 | %% that case since a transfer to it would be invalid 87 | case Pos >= riak_core_ring:num_partitions(Ring1) orelse 88 | (N =/= undefined andalso Pos >= N) of 89 | true -> TargetIdx = undefined; 90 | false -> {TargetIdx, _} = lists:nth(Pos+1, FuturePreflist) 91 | end, 92 | {CHashKey, SourceIdx, TargetIdx, N, Pos, Ring} 93 | end)). 94 | 95 | index_in_current_ring(RingSize) -> 96 | elements([I || {I,_} <- riak_core_ring:all_owners(riak_core_ring:fresh(RingSize, node()))]). 97 | 98 | current_size() -> 99 | elements([16, 32, 64, 128]). 100 | 101 | growth_factor() -> 102 | oneof([0.25, 0.5, 2, 4]). 103 | 104 | 105 | replica_position(RingSize) -> 106 | %% use a max position that could be invalid for shrinking (greater than future size) 107 | %% purposefully to generate negative cases 108 | Max = RingSize, 109 | choose(0, (Max - 1)). 110 | 111 | check_nval(RingSize, GrowthF) -> 112 | case GrowthF > 1 of 113 | %% while expanding the n-value doesn't matter 114 | true -> undefined; 115 | %% while shrinking provide a "realistic" n-value of the key 116 | false -> choose(1, trunc((RingSize * GrowthF) / 2) - 1) 117 | end. 118 | 119 | 120 | prop_future_index_failed({CHashKey, OrigIdx, TargetIdx, NValCheck, _, R}) -> 121 | <> = CHashKey, 122 | FoundTarget = riak_core_ring:future_index(CHashKey, OrigIdx, NValCheck, R), 123 | io:format("key: ~p~nsource: ~p~ncurrsize: ~p~nfuturesize: ~p~nexpected: ~p~nactual: ~p~n", 124 | [CHashInt, OrigIdx, 125 | riak_core_ring:num_partitions(R), riak_core_ring:future_num_partitions(R), 126 | TargetIdx, FoundTarget]). 127 | 128 | -endif. 129 | -------------------------------------------------------------------------------- /eqc/worker_pool_pulse.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2013 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | %% @doc Test riak_core_vnode_worker_pool's interaction with poolboy 22 | %% under PULSE. This requires that riak_core, poolboy, and this module 23 | %% be compiled with the 'PULSE' macro defined. 24 | -module(worker_pool_pulse). 25 | 26 | -behaviour(riak_core_vnode_worker). 27 | -ifdef(EQC). 28 | -include_lib("eqc/include/eqc.hrl"). 29 | -endif. 30 | -include_lib("eunit/include/eunit.hrl"). 31 | 32 | %% riak_core_vnode_worker behavior 33 | -export([init_worker/3, handle_work/3]). 34 | %% console debugging convenience 35 | -compile([export_all, nowarn_export_all]). 36 | 37 | -ifdef(PULSE). 38 | -include_lib("pulse/include/pulse.hrl"). 39 | %% have to transform the 'receive' of the work results 40 | -compile({parse_transform, pulse_instrument}). 41 | %% don't trasnform toplevel test functions 42 | -compile({pulse_skip,[{prop_any_pool,0}, 43 | {prop_small_pool,0}, 44 | {pool_test_,0}]}). 45 | -endif. 46 | 47 | %%% Worker Definition - does nothing but reply with what it is given 48 | 49 | init_worker(_VnodeIndex, Noreply, _WorkerProps) -> 50 | {ok, Noreply}. 51 | 52 | handle_work(die, _From, _State) -> 53 | exit(test_die); 54 | handle_work(Work, _From, State) -> 55 | {reply, Work, State}. 56 | 57 | %% none of these tests make sense if PULSE is not used 58 | -ifdef(PULSE). 59 | 60 | %% @doc Any amount of work should complete through any size pool. 61 | prop_any_pool() -> 62 | ?SETUP(fun setup_and_teardown/0, 63 | ?FORALL({Seed, ExtraWork, WorkList}, 64 | {pulse:seed(), 65 | frequency([{10,true},{1,false}]), 66 | list(frequency([{10,nat()}, {1,die}]))}, 67 | aggregate([{{extra_work, 68 | pool_size(ExtraWork, WorkList) < length(WorkList) }, 69 | {deaths, lists:member(die, WorkList)}}], 70 | ?WHENFAIL( 71 | io:format(user, 72 | "PoolSize: ~b~n" 73 | "WorkList: ~p~n" 74 | "Schedule: ~p~n", 75 | [pool_size(ExtraWork, WorkList), 76 | WorkList, 77 | pulse:get_schedule()]), 78 | begin 79 | PoolSize = pool_size(ExtraWork, WorkList), 80 | true == all_work_gets_done(Seed, PoolSize, WorkList) 81 | end)))). 82 | 83 | pool_size(false, WorkList) -> 84 | length(WorkList); 85 | pool_size(true, WorkList) -> 86 | case length(WorkList) div 2 of 87 | 0 -> 88 | 1; 89 | Size -> 90 | Size 91 | end. 92 | 93 | %% @doc Minimal case for the issue this test was created to probe: one 94 | %% worker, two inputs. 95 | prop_small_pool() -> 96 | ?SETUP(fun setup_and_teardown/0, 97 | ?PULSE(Result, all_work_gets_done(1, [1,2]), 98 | true == Result)). 99 | 100 | all_work_gets_done(Seed, PoolSize, WorkList) -> 101 | pulse:run_with_seed( 102 | fun() -> all_work_gets_done(PoolSize, WorkList) end, 103 | Seed). 104 | 105 | all_work_gets_done(PoolSize, WorkList) -> 106 | %% get the pool up 107 | {ok, Pool} = riak_core_vnode_worker_pool:start_link( 108 | ?MODULE, PoolSize, 10, false, []), 109 | 110 | %% send all the work 111 | [ riak_core_vnode_worker_pool:handle_work( 112 | Pool, W, {raw, N, self()}) 113 | || {N, W} <- lists:zip(lists:seq(1, length(WorkList)), WorkList) ], 114 | 115 | %% wait for all the work 116 | Results = [ receive {N, _} -> ok end 117 | || N <- lists:seq(1, length(WorkList)) ], 118 | riak_core_vnode_worker_pool:stop(Pool, normal), 119 | 120 | %% check that we got a response for every piece of work 121 | %% TODO: actually not needed, since the bug is deadlock, and will 122 | %% thus never get here 123 | length(Results) == length(WorkList). 124 | 125 | setup_and_teardown() -> 126 | error_logger:tty(false), 127 | pulse:start(), 128 | fun() -> 129 | pulse:stop(), 130 | error_logger:tty(true) 131 | end. 132 | 133 | -endif. 134 | -------------------------------------------------------------------------------- /include/riak_core.hrl: -------------------------------------------------------------------------------- 1 | -type riak_core_dict() :: dict:dict(). 2 | -type riak_core_set() :: sets:set(). -------------------------------------------------------------------------------- /include/riak_core_bg_manager.hrl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2013 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | -type bg_lock() :: any(). 20 | -type bg_token() :: any(). 21 | -type bg_resource() :: bg_token() | bg_lock(). 22 | -type bg_resource_type() :: lock | token. 23 | 24 | -type bg_meta() :: undefined | {atom(), any()}. %% meta data to associate with a lock/token 25 | -type bg_period() :: pos_integer(). %% token refill period in milliseconds 26 | -type bg_count() :: pos_integer(). %% token refill tokens to count at each refill period 27 | -type bg_rate() :: undefined | {bg_period(), bg_count()}. %% token refill rate 28 | -type bg_concurrency_limit() :: non_neg_integer() | infinity. %% max lock concurrency allowed 29 | -type bg_consumer() :: {undefined | pid(), bg_meta()}. %% a consumer of a resource 30 | 31 | %% Results of a "ps" of live given or blocked locks/tokens 32 | -record(bg_stat_live, 33 | { 34 | resource :: bg_resource(), %% resource name, e.g. 'aae_hashtree_lock' 35 | type :: bg_resource_type(), %% resource type, e.g. 'lock' 36 | owner :: bg_consumer() %% this consumer has the lock or token 37 | }). 38 | -type bg_stat_live() :: #bg_stat_live{}. 39 | 40 | -define(BG_INFO_ETS_TABLE, background_mgr_info_table). %% name of lock/token manager info ETS table 41 | -define(BG_INFO_ETS_OPTS, [public, %% creation time properties of info ETS table 42 | named_table, 43 | set]). 44 | 45 | -define(BG_ENTRY_ETS_TABLE, background_mgr_entry_table). %% name of lock/token manager entry ETS table 46 | -define(BG_ENTRY_ETS_OPTS, [public, %% creation time properties of entry ETS table 47 | named_table, 48 | bag]). 49 | 50 | 51 | -------------------------------------------------------------------------------- /include/riak_core_bucket_type.hrl: -------------------------------------------------------------------------------- 1 | -define(BUCKET_TYPE_PREFIX, {core, bucket_types}). 2 | -define(DEFAULT_TYPE, <<"default">>). 3 | 4 | -------------------------------------------------------------------------------- /include/riak_core_dtrace.hrl: -------------------------------------------------------------------------------- 1 | %% NOTE: Coordinate ?DTRACE_TAG_KEY with riak_core_dtrace.erl 2 | -define(DTRACE_TAG_KEY, '**DTRACE*TAG*KEY**'). 3 | 4 | %% Erlang tracing-related funnel functions for DTrace/SystemTap. When 5 | %% using Redbug or other Erlang tracing framework, trace these 6 | %% functions. Include these here to avoid copy/pasting to every 7 | %% module. 8 | 9 | dtrace(_BKey, Category, Ints, Strings) -> 10 | riak_core_dtrace:dtrace(Category, Ints, Strings). 11 | 12 | %% Internal functions, not so interesting for Erlang tracing. 13 | 14 | dtrace_int(Category, Ints, Strings) -> 15 | dtrace(get(?DTRACE_TAG_KEY), Category, Ints, Strings). 16 | -------------------------------------------------------------------------------- /include/riak_core_handoff.hrl: -------------------------------------------------------------------------------- 1 | -type riak_core_handoff_dict() :: dict:dict(). 2 | 3 | -define(PT_MSG_INIT, 0). 4 | -define(PT_MSG_OBJ, 1). 5 | -define(PT_MSG_OLDSYNC, 2). 6 | -define(PT_MSG_SYNC, 3). 7 | -define(PT_MSG_CONFIGURE, 4). 8 | -define(PT_MSG_BATCH, 5). 9 | 10 | -record(ho_stats, 11 | { 12 | interval_end :: erlang:timestamp(), 13 | last_update = os:timestamp() :: erlang:timestamp(), 14 | objs=0 :: non_neg_integer(), 15 | bytes=0 :: non_neg_integer() 16 | }). 17 | 18 | -type ho_stats() :: #ho_stats{}. 19 | -type ho_type() :: ownership | hinted | repair | resize. 20 | -type predicate() :: fun((any()) -> boolean()). 21 | 22 | -type index() :: chash:index_as_int(). 23 | -type mod_src_tgt() :: {module(), index(), index()} | {undefined, undefined, undefined}. 24 | -type mod_partition() :: {module(), index()}. 25 | 26 | -record(handoff_status, 27 | { mod_src_tgt :: mod_src_tgt(), 28 | src_node :: node(), 29 | target_node :: node(), 30 | direction :: inbound | outbound, 31 | transport_pid :: pid(), 32 | transport_mon :: reference(), 33 | timestamp :: tuple(), 34 | status :: any(), 35 | stats :: riak_core_handoff_dict(), 36 | vnode_pid :: pid() | undefined, 37 | vnode_mon :: reference() | undefined, 38 | type = undefined :: ho_type() | undefined, 39 | req_origin :: node(), 40 | filter_mod_fun :: {module(), atom()} | undefined, 41 | size = {0, objects} :: {function(), dynamic} | {non_neg_integer(), bytes | objects} 42 | }). 43 | -type handoff_status() :: #handoff_status{}. 44 | 45 | -type known_handoff() :: {{module(), index()}, 46 | {ho_type()|'delete', 47 | 'inbound'|'outbound'|'local', 48 | node()|'$resize'|'$delete'}}. 49 | -------------------------------------------------------------------------------- /include/riak_core_metadata.hrl: -------------------------------------------------------------------------------- 1 | -type metadata_prefix() :: {binary() | atom(), binary() | atom()}. 2 | -type metadata_key() :: any(). 3 | -type metadata_pkey() :: {metadata_prefix(), metadata_key()}. 4 | -type metadata_value() :: any(). 5 | -type metadata_tombstone() :: '$deleted'. 6 | -type metadata_resolver() :: fun((metadata_value() | metadata_tombstone(), 7 | metadata_value() | metadata_tombstone()) -> metadata_value()). 8 | -type metadata_modifier() :: fun(([metadata_value() | metadata_tombstone()] | undefined) -> 9 | metadata_value()). 10 | -type metadata_object() :: {metadata, dvvset:clock()}. 11 | -type metadata_context() :: dvvset:vector(). 12 | 13 | -record(metadata_broadcast, { 14 | pkey :: metadata_pkey(), 15 | obj :: metadata_object() 16 | }). 17 | -type metadata_broadcast() :: #metadata_broadcast{}. 18 | -------------------------------------------------------------------------------- /include/riak_core_ring.hrl: -------------------------------------------------------------------------------- 1 | -define(LEGACY_RING_VSN, 1). 2 | -define(CURRENT_RING_VSN, 2). 3 | -------------------------------------------------------------------------------- /include/riak_core_vnode.hrl: -------------------------------------------------------------------------------- 1 | -type sender_type() :: fsm | server | raw. 2 | -type sender() :: {sender_type(), reference() | tuple(), pid()} | 3 | %% TODO: Double-check that these special cases are kosher 4 | {server, undefined, undefined} | % special case in 5 | % riak_core_vnode_master.erl 6 | {fsm, undefined, pid()} | % special case in 7 | % riak_kv_util:make_request/2.erl 8 | ignore. 9 | -type partition() :: chash:index_as_int(). 10 | -type vnode_req() :: term(). 11 | -type keyspaces() :: [{partition(), [partition()]}]. 12 | 13 | -record(riak_vnode_req_v1, { 14 | index :: partition() | undefined, 15 | sender=ignore :: sender(), 16 | request :: vnode_req()}). 17 | 18 | -record(riak_coverage_req_v1, { 19 | index :: partition(), 20 | keyspaces :: keyspaces(), 21 | sender=ignore :: sender(), 22 | request :: vnode_req()}). 23 | 24 | -record(riak_core_fold_req_v1, { 25 | foldfun :: fun(), 26 | acc0 :: term()}). 27 | -record(riak_core_fold_req_v2, { 28 | foldfun :: fun(), 29 | acc0 :: term(), 30 | forwardable :: boolean(), 31 | opts = [] :: list()}). 32 | 33 | -define(VNODE_REQ, #riak_vnode_req_v1). 34 | -define(COVERAGE_REQ, #riak_coverage_req_v1). 35 | -define(FOLD_REQ, #riak_core_fold_req_v2). 36 | 37 | -type handoff_dest() :: {riak_core_handoff_manager:ho_type(), {partition(), node()}}. 38 | -------------------------------------------------------------------------------- /rebar.config: -------------------------------------------------------------------------------- 1 | {minimum_otp_vsn, "22.0"}. 2 | 3 | {erl_first_files, ["src/gen_nb_server.erl", "src/riak_core_gen_server.erl", 4 | "src/riak_core_stat_xform"]}. 5 | 6 | {cover_enabled, true}. 7 | 8 | {erl_opts, [warnings_as_errors, debug_info]}. 9 | 10 | {edoc_opts, [{preprocess, true}]}. 11 | 12 | {eunit_opts, [verbose]}. 13 | 14 | {xref_checks,[undefined_function_calls,undefined_functions,locals_not_used]}. 15 | 16 | {plugins, [{eqc_rebar, {git, "https://github.com/Quviq/eqc-rebar", {branch, "master"}}}]}. 17 | 18 | {deps, [ 19 | {poolboy, {git, "https://github.com/basho/poolboy.git", {branch, "develop-3.2"}}}, 20 | {riak_sysmon, {git, "https://github.com/basho/riak_sysmon.git", {branch, "develop"}}}, 21 | {clique, {git, "https://github.com/basho/clique.git", {branch, "develop"}}}, 22 | {eleveldb, {git, "https://github.com/basho/eleveldb.git", {branch, "develop"}}}, 23 | {riak_ensemble, {git, "https://github.com/basho/riak_ensemble", {branch, "develop"}}}, 24 | {pbkdf2, {git, "https://github.com/basho/erlang-pbkdf2.git", {branch, "develop"}}}, 25 | {cluster_info, {git, "https://github.com/basho/cluster_info.git", {branch, "develop"}}}, 26 | {exometer_core, {git, "https://github.com/Feuerlabs/exometer_core.git", {tag, "1.6.1"}}}, 27 | {basho_stats, {git, "https://github.com/basho/basho_stats.git", {branch, "develop"}}} 28 | ]}. 29 | 30 | {dialyzer, [{plt_apps, all_deps}]}. 31 | 32 | {profiles, [ 33 | {test, [{deps, [meck]}, {erl_opts, [nowarn_export_all]}]}, 34 | {eqc, [{deps, [meck]}, {erl_opts, [{d, 'EQC'}, nowarn_export_all]}]}, 35 | {gha, [{erl_opts, [{d, 'GITHUBEXCLUDE'}]}]} 36 | ]}. 37 | -------------------------------------------------------------------------------- /rebar3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/basho/riak_core/25d9a6fa917eb8a2e95795d64eb88d7ad384ed88/rebar3 -------------------------------------------------------------------------------- /src/app_helper.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riak_core: Core Riak Application 4 | %% 5 | %% Copyright (c) 2007-2012 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | -module(app_helper). 24 | 25 | -export([get_env/1, 26 | get_env/2, 27 | get_env/3, 28 | get_prop_or_env/3, 29 | get_prop_or_env/4, 30 | try_envs/1, 31 | try_envs/2]). 32 | 33 | -ifdef(TEST). 34 | -include_lib("eunit/include/eunit.hrl"). 35 | -endif. 36 | 37 | %% =================================================================== 38 | %% Public API 39 | %% =================================================================== 40 | 41 | %% @spec get_env(App :: atom()) -> [{Key :: atom(), Value :: term()}] 42 | %% @doc Retrieve all Key/Value pairs in the env for the specified app. 43 | get_env(App) -> 44 | application:get_all_env(App). 45 | 46 | %% @spec get_env(App :: atom(), Key :: atom()) -> term() 47 | %% @doc The official way to get a value from the app's env. 48 | %% Will return the 'undefined' atom if that key is unset. 49 | get_env(App, Key) -> 50 | get_env(App, Key, undefined). 51 | 52 | %% @spec get_env(App :: atom(), Key :: atom(), Default :: term()) -> term() 53 | %% @doc The official way to get a value from this application's env. 54 | %% Will return Default if that key is unset. 55 | get_env(App, Key, Default) -> 56 | case application:get_env(App, Key) of 57 | {ok, Value} -> 58 | Value; 59 | _ -> 60 | Default 61 | end. 62 | 63 | %% @doc Retrieve value for Key from Properties if it exists, otherwise 64 | %% return from the application's env. 65 | -spec get_prop_or_env(atom(), [{atom(), term()}], atom()) -> term(). 66 | get_prop_or_env(Key, Properties, App) -> 67 | get_prop_or_env(Key, Properties, App, undefined). 68 | 69 | %% @doc Return the value for Key in Properties if it exists, otherwise return 70 | %% the value from the application's env, or Default. 71 | -spec get_prop_or_env(atom(), [{atom(), term()}], atom(), term()) -> term(). 72 | get_prop_or_env(Key, Properties, App, Default) -> 73 | case proplists:get_value(Key, Properties) of 74 | undefined -> 75 | get_env(App, Key, Default); 76 | Value -> 77 | Value 78 | end. 79 | 80 | %% @doc Like `get_env' but try multiple `{App, Key}' combos before 81 | %% returning `{default, Default}'. The return value is `{App, 82 | %% Key, Value}' so that the caller may distinguish where the 83 | %% value came from. This is useful for scenarios when the config 84 | %% app/key has changed between releases and you need to check for 85 | %% both. 86 | -spec try_envs([{atom(), atom()}], term()) -> {atom(), atom(), term()}. 87 | try_envs([{App, Key}|T], Default) -> 88 | case get_env(App, Key) of 89 | undefined -> 90 | try_envs(T, Default); 91 | Value -> 92 | {App, Key, Value} 93 | end; 94 | try_envs([], Default) -> 95 | {default, Default}. 96 | 97 | try_envs(Pairs) -> 98 | try_envs(Pairs, undefined). 99 | 100 | %% =================================================================== 101 | %% EUnit tests 102 | %% =================================================================== 103 | -ifdef(TEST). 104 | 105 | app_helper_test_() -> 106 | { setup, 107 | fun setup/0, 108 | fun cleanup/1, 109 | [ 110 | fun get_prop_or_env_default_value_test_case/0, 111 | fun get_prop_or_env_undefined_value_test_case/0, 112 | fun get_prop_or_env_from_env_test_case/0, 113 | fun get_prop_or_env_from_prop_test_case/0, 114 | fun get_prop_or_env_from_prop_with_default_test_case/0, 115 | fun try_envs_test_case/0 116 | ] 117 | }. 118 | 119 | setup() -> 120 | application:set_env(bogus_app, envkeyone, value), 121 | application:set_env(bogus_app, envkeytwo, valuetwo). 122 | 123 | cleanup(_Ctx) -> 124 | ok. 125 | 126 | get_prop_or_env_default_value_test_case() -> 127 | ?assertEqual(default, get_prop_or_env(key, [], bogus, default)). 128 | 129 | get_prop_or_env_undefined_value_test_case() -> 130 | ?assertEqual(undefined, get_prop_or_env(key, [], bogus)). 131 | 132 | get_prop_or_env_from_env_test_case() -> 133 | ?assertEqual(value, get_prop_or_env(envkeyone, [], bogus_app)). 134 | 135 | get_prop_or_env_from_prop_test_case() -> 136 | Properties = [{envkeyone, propvalue}], 137 | ?assertEqual(propvalue, get_prop_or_env(envkeyone, Properties, bogus_app)). 138 | 139 | get_prop_or_env_from_prop_with_default_test_case() -> 140 | Properties = [{envkeyone, propvalue}], 141 | ?assertEqual(propvalue, get_prop_or_env(envkeyone, Properties, bogus_app, default)). 142 | 143 | try_envs_test_case() -> 144 | Val = try_envs([{noapp, nokey}, {bogus_app, envkeyone}], failed), 145 | ?assertEqual({bogus_app, envkeyone, value}, Val), 146 | Val2 = try_envs([{bogus_app, envkeytwo}, {noapp, nokey}], failed), 147 | ?assertEqual({bogus_app, envkeytwo, valuetwo}, Val2), 148 | Val3 = try_envs([{noapp, nokey}, {blah, blah}], default), 149 | ?assertEqual({default, default}, Val3). 150 | 151 | -endif. 152 | -------------------------------------------------------------------------------- /src/process_proxy.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved. 3 | %% 4 | %% This file is provided to you under the Apache License, 5 | %% Version 2.0 (the "License"); you may not use this file 6 | %% except in compliance with the License. You may obtain 7 | %% a copy of the License at 8 | %% 9 | %% http://www.apache.org/licenses/LICENSE-2.0 10 | %% 11 | %% Unless required by applicable law or agreed to in writing, 12 | %% software distributed under the License is distributed on an 13 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | %% KIND, either express or implied. See the License for the 15 | %% specific language governing permissions and limitations 16 | %% under the License. 17 | %% 18 | %% ------------------------------------------------------------------- 19 | -module(process_proxy). 20 | -export([start_link/2, init/1, stop/1]). 21 | 22 | start_link(RegName, ProxyTo) -> 23 | proc_lib:start_link(?MODULE, init, [[self(), RegName, ProxyTo]]). 24 | 25 | init([ParentPid, RegName, ProxyTo]) -> 26 | erlang:register(RegName, self()), 27 | proc_lib:init_ack(ParentPid, {ok, self()}), 28 | loop(ProxyTo). 29 | 30 | stop(Name) -> 31 | Name ! stop. 32 | 33 | loop(ProxyTo) -> 34 | receive 35 | stop -> 36 | exit(normal); 37 | M -> 38 | ProxyTo ! M, 39 | loop(ProxyTo) 40 | end. 41 | -------------------------------------------------------------------------------- /src/riak_core.app.src: -------------------------------------------------------------------------------- 1 | %% -*- tab-width: 4;erlang-indent-level: 4;indent-tabs-mode: nil -*- 2 | %% ex: ts=4 sw=4 et 3 | {application, riak_core, 4 | [ 5 | {description, "Riak Core"}, 6 | {vsn, git}, 7 | {modules, []}, 8 | {registered, []}, 9 | {included_applications, [riak_ensemble]}, 10 | {applications, [ 11 | kernel, 12 | stdlib, 13 | sasl, 14 | crypto, 15 | ssl, 16 | runtime_tools, 17 | inets, 18 | compiler, 19 | riak_sysmon, 20 | os_mon, 21 | eleveldb, 22 | pbkdf2, 23 | poolboy, 24 | exometer_core, 25 | clique, 26 | cluster_info, 27 | basho_stats 28 | ]}, 29 | {mod, {riak_core_app, []}}, 30 | {env, [ 31 | %% Cluster name 32 | {cluster_name, "default"}, 33 | 34 | %% Default location for ring, cluster and other data files 35 | {platform_data_dir, "data"}, 36 | 37 | %% Default ring creation size. Make sure it is a power of 2, 38 | %% e.g. 16, 32, 64, 128, 256, 512 etc 39 | {ring_creation_size, 64}, 40 | 41 | %% Default gossip interval (milliseconds) 42 | {gossip_interval, 60000}, 43 | 44 | %% Target N value 45 | {target_n_val, 4}, 46 | 47 | %% Default claims functions 48 | {wants_claim_fun, 49 | {riak_core_membership_claim, default_wants_claim}}, 50 | {choose_claim_fun, 51 | {riak_core_membership_claim, default_choose_claim}}, 52 | 53 | %% Vnode inactivity timeout (how often to check if fallback vnodes 54 | %% should return their data) in ms. 55 | {vnode_inactivity_timeout, 60000}, 56 | 57 | %% Number of VNodes allowed to do handoff concurrently. 58 | {handoff_concurrency, 2}, 59 | 60 | %% Disable Nagle on HTTP sockets 61 | {disable_http_nagle, true}, 62 | 63 | %% Handoff IP/port 64 | {handoff_port, 8099}, 65 | {handoff_ip, "0.0.0.0"}, 66 | 67 | %% Disterl buffer sizes in bytes. 68 | %% These sizes (3*128*1024 & 6*128*1024) were 69 | %% derived from a limited amount of testing in a 70 | %% 10GE environment, and may need tuning for your 71 | %% network and workload. In particular they're likely 72 | %% too small to be optimal for larger object sizes. 73 | {dist_send_buf_size, 393216}, 74 | {dist_recv_buf_size, 786432}, 75 | 76 | %% Exometer defaults 77 | {exometer_defaults, 78 | [ 79 | {['_'], histogram, [{options, 80 | [{histogram_module, exometer_slot_slide}, 81 | {keep_high, 500}]} 82 | ]} 83 | ]} 84 | ]} 85 | ]}. 86 | -------------------------------------------------------------------------------- /src/riak_core_app.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riak_core: Core Riak Application 4 | %% 5 | %% Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | -module(riak_core_app). 24 | 25 | -behaviour(application). 26 | 27 | %% Application callbacks 28 | -export([start/2, stop/1]). 29 | 30 | -include_lib("kernel/include/logger.hrl"). 31 | 32 | %% =================================================================== 33 | %% Application callbacks 34 | %% =================================================================== 35 | 36 | start(_StartType, _StartArgs) -> 37 | %% Don't add our system_monitor event handler here. Instead, let 38 | %% riak_core_sysmon_minder start it, because that process can act 39 | %% on any handler crash notification, whereas we cannot. 40 | 41 | case application:get_env(riak_core, delayed_start) of 42 | {ok, Delay} -> 43 | ?LOG_INFO("Delaying riak_core startup as requested"), 44 | timer:sleep(Delay); 45 | _ -> 46 | ok 47 | end, 48 | 49 | %% Validate that the ring state directory exists 50 | riak_core_util:start_app_deps(riak_core), 51 | RingStateDir = app_helper:get_env(riak_core, ring_state_dir), 52 | case filelib:ensure_dir(filename:join(RingStateDir, "dummy")) of 53 | ok -> 54 | ok; 55 | {error, RingReason} -> 56 | ?LOG_CRITICAL( 57 | "Ring state directory ~p does not exist, " 58 | "and could not be created: ~p", 59 | [RingReason]), 60 | throw({error, invalid_ring_state_dir}) 61 | end, 62 | 63 | %% Register our cluster_info app callback modules, with catch if 64 | %% the app is missing or packaging is broken. 65 | catch cluster_info:register_app(riak_core_cinfo_core), 66 | 67 | %% add these defaults now to supplement the set that may have been 68 | %% configured in app.config 69 | riak_core_bucket:append_bucket_defaults(riak_core_bucket_type:defaults(default_type)), 70 | 71 | %% Spin up the supervisor; prune ring files as necessary 72 | case riak_core_sup:start_link() of 73 | {ok, Pid} -> 74 | riak_core:register(riak_core, [{stat_mod, riak_core_stat}, 75 | {permissions, [get_bucket, 76 | set_bucket, 77 | get_bucket_type, 78 | set_bucket_type]}]), 79 | ok = riak_core_ring_events:add_guarded_handler(riak_core_ring_handler, []), 80 | 81 | riak_core_capability:register({riak_core, vnode_routing}, 82 | [proxy, legacy], 83 | legacy, 84 | {riak_core, 85 | legacy_vnode_routing, 86 | [{true, legacy}, {false, proxy}]}), 87 | riak_core_capability:register({riak_core, staged_joins}, 88 | [true, false], 89 | false), 90 | riak_core_capability:register({riak_core, resizable_ring}, 91 | [true, false], 92 | false), 93 | riak_core_capability:register({riak_core, fold_req_version}, 94 | [v2, v1], 95 | v1), 96 | riak_core_capability:register({riak_core, security}, 97 | [true, false], 98 | false), 99 | riak_core_capability:register({riak_core, bucket_types}, 100 | [true, false], 101 | false), 102 | riak_core_capability:register({riak_core, net_ticktime}, 103 | [true, false], 104 | false), 105 | 106 | riak_core_cli_registry:load_schema(), 107 | riak_core_cli_registry:register_node_finder(), 108 | riak_core_cli_registry:register_cli(), 109 | 110 | riak_core_throttle:init(), 111 | 112 | {ok, Pid}; 113 | {error, Reason} -> 114 | {error, Reason} 115 | end. 116 | 117 | stop(_State) -> 118 | ?LOG_INFO("Stopped application riak_core.\n", []), 119 | ok. 120 | -------------------------------------------------------------------------------- /src/riak_core_broadcast_handler.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2013 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | -module(riak_core_broadcast_handler). 21 | 22 | %% Return a two-tuple of message id and payload from a given broadcast 23 | -callback broadcast_data(any()) -> {any(), any()}. 24 | 25 | %% Given the message id and payload, merge the message in the local state. 26 | %% If the message has already been received return `false', otherwise return `true' 27 | -callback merge(any(), any()) -> boolean(). 28 | 29 | %% Return true if the message (given the message id) has already been received. 30 | %% `false' otherwise 31 | -callback is_stale(any()) -> boolean(). 32 | 33 | %% Return the message associated with the given message id. In some cases a message 34 | %% has already been sent with information that subsumes the message associated with the given 35 | %% message id. In this case, `stale' is returned. 36 | -callback graft(any()) -> stale | {ok, any()} | {error, any()}. 37 | 38 | %% Trigger an exchange between the local handler and the handler on the given node. 39 | %% How the exchange is performed is not defined but it should be performed as a background 40 | %% process and ensure that it delivers any messages missing on either the local or remote node. 41 | %% The exchange does not need to account for messages in-flight when it is started or broadcast 42 | %% during its operation. These can be taken care of in future exchanges. 43 | -callback exchange(node()) -> {ok, pid()} | {error, term()}. 44 | -------------------------------------------------------------------------------- /src/riak_core_cinfo_core.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Riak: A lightweight, decentralized key-value store. 4 | %% 5 | %% Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | -module(riak_core_cinfo_core). 23 | 24 | -export([cluster_info_init/0, cluster_info_generator_funs/0]). 25 | 26 | %% @spec () -> term() 27 | %% @doc Required callback function for cluster_info: initialization. 28 | %% 29 | %% This function doesn't have to do anything. 30 | 31 | cluster_info_init() -> 32 | ok. 33 | 34 | %% @spec () -> list({string(), fun()}) 35 | %% @doc Required callback function for cluster_info: return list of 36 | %% {NameForReport, FunOfArity_1} tuples to generate ASCII/UTF-8 37 | %% formatted reports. 38 | 39 | cluster_info_generator_funs() -> 40 | [ 41 | {"Riak Core config files", fun config_files/1}, 42 | {"Riak Core vnode modules", fun vnode_modules/1}, 43 | {"Riak Core ring", fun get_my_ring/1}, 44 | {"Riak Core latest ring file", fun latest_ringfile/1}, 45 | {"Riak Core active partitions", fun active_partitions/1} 46 | ]. 47 | 48 | vnode_modules(CPid) -> % CPid is the data collector's pid. 49 | cluster_info:format(CPid, "~p\n", [riak_core:vnode_modules()]). 50 | 51 | get_my_ring(CPid) -> 52 | {ok, Ring} = riak_core_ring_manager:get_my_ring(), 53 | cluster_info:format(CPid, "~p\n", [Ring]). 54 | 55 | latest_ringfile(CPid) -> 56 | {ok, Path} = riak_core_ring_manager:find_latest_ringfile(), 57 | {ok, Contents} = file:read_file(Path), 58 | cluster_info:format(CPid, "Latest ringfile: ~s\n", [Path]), 59 | cluster_info:format(CPid, "File contents:\n~p\n", [binary_to_term(Contents)]). 60 | 61 | active_partitions(CPid) -> 62 | Vnodes = [{Mod, Idx} || {Mod, Idx, _Pid} <- riak_core_vnode_manager:all_vnodes()], 63 | Partitions = lists:foldl(fun({_,P}, Ps) -> 64 | ordsets:add_element(P, Ps) 65 | end, ordsets:new(), Vnodes), 66 | cluster_info:format(CPid, "~p\n", [Partitions]). 67 | 68 | config_files(C) -> 69 | {ok, [[AppPath]]} = init:get_argument(config), 70 | EtcDir = filename:dirname(AppPath), 71 | VmPath = filename:join(EtcDir, "vm.args"), 72 | [begin 73 | cluster_info:format(C, "File: ~s\n", [os:cmd("ls -l " ++ File)]), 74 | {ok, FileBin} = file:read_file(File), 75 | cluster_info:format(C, "File contents:\n~s\n", [FileBin]) 76 | end || File <- [AppPath, VmPath]]. 77 | 78 | -------------------------------------------------------------------------------- /src/riak_core_cli_registry.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2014 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | %% @doc This module tracks registrations for clique integrations. 21 | 22 | -module(riak_core_cli_registry). 23 | 24 | -define(CLI_MODULES, [ 25 | riak_core_cluster_cli, 26 | riak_core_handoff_cli 27 | ]). 28 | 29 | -export([ 30 | register_node_finder/0, 31 | register_cli/0, 32 | load_schema/0 33 | ]). 34 | 35 | -spec register_node_finder() -> true. 36 | register_node_finder() -> 37 | F = fun() -> 38 | {ok, MyRing} = riak_core_ring_manager:get_my_ring(), 39 | riak_core_ring:all_members(MyRing) 40 | end, 41 | clique:register_node_finder(F). 42 | 43 | -spec register_cli() -> ok. 44 | register_cli() -> 45 | clique:register(?CLI_MODULES). 46 | 47 | -spec load_schema() -> ok. 48 | load_schema() -> 49 | case application:get_env(riak_core, schema_dirs) of 50 | {ok, Directories} -> 51 | ok = clique_config:load_schema(Directories); 52 | _ -> 53 | ok = clique_config:load_schema([code:lib_dir()]) 54 | end. 55 | -------------------------------------------------------------------------------- /src/riak_core_dist_mon.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Disterl socket buffer size monitor 4 | %% 5 | %% Copyright (c) 2013 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | -module(riak_core_dist_mon). 24 | 25 | -export([start_link/0, set_dist_buf_sizes/2, get_riak_env_vars/0]). 26 | 27 | %% gen_server callbacks 28 | -behavior(gen_server). 29 | -export([init/1, handle_call/3, handle_cast/2, handle_info/2, 30 | terminate/2, code_change/3]). 31 | 32 | -include_lib("kernel/include/logger.hrl"). 33 | 34 | -record(state, { 35 | sndbuf :: non_neg_integer(), 36 | recbuf :: non_neg_integer() 37 | }). 38 | 39 | %% provided for testing convenience and debugging. 40 | set_dist_buf_sizes(SndBuf, RecBuf) 41 | when (is_integer(SndBuf) andalso SndBuf > 0) andalso 42 | (is_integer(RecBuf) andalso RecBuf > 0) -> 43 | gen_server:call(?MODULE, {set_dist_buf_sizes, SndBuf, RecBuf}). 44 | 45 | 46 | start_link() -> 47 | gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). 48 | 49 | init(_) -> 50 | %% register for monitor events so we can adjust buffers on 51 | %% new connections when they're started. 52 | ok = net_kernel:monitor_nodes(true, [{node_type, visible}, nodedown_reason]), 53 | {SndBuf, RecBuf} = get_riak_env_vars(), 54 | DistCtrl = erlang:system_info(dist_ctrl), 55 | %% make sure that buffers are correct on existing connections. 56 | _ = [set_port_buffers(Port, SndBuf, RecBuf) 57 | || {_Node, Port} <- DistCtrl], 58 | {ok, #state{sndbuf=SndBuf, recbuf=RecBuf}}. 59 | 60 | get_riak_env_vars() -> 61 | {ok, SndBuf} = application:get_env(riak_core, dist_send_buf_size), 62 | {ok, RecBuf} = application:get_env(riak_core, dist_recv_buf_size), 63 | {SndBuf, RecBuf}. 64 | 65 | handle_call({set_dist_buf_sizes, SndBuf, RecBuf}, _From, State) -> 66 | _ = [set_port_buffers(Port, SndBuf, RecBuf) 67 | || {_Node, Port} <- erlang:system_info(dist_ctrl)], 68 | {reply, ok, State#state{sndbuf=SndBuf, recbuf=RecBuf}}; 69 | handle_call(Msg, _From, State) -> 70 | ?LOG_WARNING("unknown call message received: ~p", [Msg]), 71 | {noreply, State}. 72 | 73 | handle_cast(Msg, State) -> 74 | ?LOG_WARNING("unknown cast message received: ~p", [Msg]), 75 | {noreply, State}. 76 | 77 | 78 | handle_info({nodeup, Node, _InfoList}, #state{sndbuf=SndBuf, 79 | recbuf=RecBuf} = State) -> 80 | DistCtrl = erlang:system_info(dist_ctrl), 81 | case proplists:get_value(Node, DistCtrl) of 82 | undefined -> 83 | ?LOG_ERROR("Could not get dist for ~p\n~p\n", [Node, DistCtrl]), 84 | {noreply, State}; 85 | Port -> 86 | ok = set_port_buffers(Port, SndBuf, RecBuf), 87 | {noreply, State} 88 | end; 89 | handle_info({nodedown, _Node, _InfoList}, State) -> 90 | %% don't think we need to do anything here 91 | {noreply, State}; 92 | handle_info(Msg, State) -> 93 | ?LOG_WARNING("unknown info message received: ~p", [Msg]), 94 | {noreply, State}. 95 | 96 | terminate(_Reason, _State) -> 97 | ok. 98 | 99 | code_change(_OldVsn, State, _Extra) -> 100 | {ok, State}. 101 | 102 | %% private 103 | 104 | set_port_buffers(Port, SndBuf, RecBuf) -> 105 | inet:setopts(Port, [{sndbuf, SndBuf}, 106 | {recbuf, RecBuf}]). 107 | -------------------------------------------------------------------------------- /src/riak_core_eventhandler_guard.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riak_core_eventhandler_guard: Guard process for persistent event handlers. 4 | %% 5 | %% Copyright (c) 2007-2011 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | -module(riak_core_eventhandler_guard). 23 | -behaviour(gen_server). 24 | -export([start_link/3, start_link/4]). 25 | -export([init/1, handle_call/3, handle_cast/2, handle_info/2, 26 | terminate/2, code_change/3]). 27 | -record(state, {handlermod, handler, exitfun}). 28 | 29 | start_link(HandlerMod, Handler, Args) -> 30 | start_link(HandlerMod, Handler, Args, undefined). 31 | 32 | start_link(HandlerMod, Handler, Args, ExitFun) -> 33 | gen_server:start_link(?MODULE, [HandlerMod, Handler, Args, ExitFun], []). 34 | 35 | init([HandlerMod, Handler, Args, ExitFun]) -> 36 | ok = gen_event:add_sup_handler(HandlerMod, Handler, Args), 37 | {ok, #state{handlermod=HandlerMod, handler=Handler, exitfun=ExitFun}}. 38 | 39 | handle_call(_Request, _From, State) -> {reply, ok, State}. 40 | 41 | handle_cast(_Msg, State) -> {noreply, State}. 42 | 43 | 44 | handle_info({gen_event_EXIT, _Handler, shutdown}, State) -> 45 | {stop, normal, State}; 46 | handle_info({gen_event_EXIT, _Handler, normal}, State) -> 47 | {stop, normal, State}; 48 | handle_info({gen_event_EXIT, Handler, _Reason}, State=#state{exitfun=undefined}) -> 49 | {stop, {gen_event_EXIT, Handler}, State}; 50 | handle_info({gen_event_EXIT, Handler, Reason}, State=#state{exitfun=ExitFun}) -> 51 | ExitFun(Handler, Reason), 52 | {stop, {gen_event_EXIT, Handler}, State}; 53 | handle_info(_Info, State) -> 54 | {noreply, State}. 55 | 56 | terminate(_Reason, #state{}) -> 57 | ok. 58 | 59 | code_change(_OldVsn, State, _Extra) -> {ok, State}. 60 | 61 | -------------------------------------------------------------------------------- /src/riak_core_eventhandler_sup.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riak_core_eventhandler_sup: supervise minder processes for gen_event handlers 4 | %% 5 | %% Copyright (c) 2007-2011 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | %% @doc supervise riak_core_eventhandler_guard processes 24 | -module(riak_core_eventhandler_sup). 25 | -behaviour(supervisor). 26 | -export([start_link/0, init/1]). 27 | -export([start_guarded_handler/3, start_guarded_handler/4, stop_guarded_handler/3]). 28 | 29 | start_guarded_handler(HandlerMod, Handler, Args) -> 30 | start_guarded_handler(HandlerMod, Handler, Args, undefined). 31 | 32 | start_guarded_handler(HandlerMod, Handler, Args, ExitFun) -> 33 | case supervisor:start_child(?MODULE, handler_spec(HandlerMod, Handler, Args, ExitFun)) of 34 | {ok, _Pid} -> ok; 35 | Other -> Other 36 | end. 37 | 38 | stop_guarded_handler(HandlerMod, Handler, Args) -> 39 | case lists:member(Handler, gen_event:which_handlers(HandlerMod)) of 40 | true -> 41 | case gen_event:delete_handler(HandlerMod, Handler, Args) of 42 | {error, module_not_found} -> 43 | {error, module_not_found}; 44 | O -> 45 | Id = {HandlerMod, Handler}, 46 | ok = supervisor:terminate_child(?MODULE, Id), 47 | ok = supervisor:delete_child(?MODULE, Id), 48 | O 49 | end; 50 | false -> 51 | {error, module_not_found} 52 | end. 53 | 54 | handler_spec(HandlerMod, Handler, Args, ExitFun) -> 55 | {{HandlerMod, Handler}, 56 | {riak_core_eventhandler_guard, start_link, [HandlerMod, Handler, Args, ExitFun]}, 57 | transient, 5000, worker, [riak_core_eventhandler_guard]}. 58 | 59 | start_link() -> 60 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 61 | 62 | %% @private 63 | init([]) -> 64 | {ok, {{one_for_one, 10, 10}, []}}. 65 | 66 | 67 | -------------------------------------------------------------------------------- /src/riak_core_exo_monitor.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2014 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This Source Code Form is subject to the terms of the Mozilla Public 6 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 7 | %% file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | %% 9 | %% ------------------------------------------------------------------- 10 | %% 11 | %% @doc Legacy mapping module folsom metrics to exometer 12 | %% 13 | %% @end 14 | -module(riak_core_exo_monitor). 15 | -behaviour(exometer_folsom_monitor). 16 | -behaviour(exometer_entry). 17 | 18 | -export([copy_folsom/3]). 19 | -export([behaviour/0, 20 | delete/3, 21 | get_datapoints/3, 22 | get_value/4, 23 | new/3, 24 | reset/3, 25 | sample/3, 26 | setopts/3, 27 | update/4]). 28 | 29 | behaviour() -> 30 | entry. 31 | 32 | copy_folsom(Name, Type, Opts) when is_tuple(Name) -> 33 | Prefix = riak_core_stat:prefix(), 34 | {[Prefix|tuple_to_list(Name)], ad_hoc, [{folsom_name, Name}, 35 | {module, ?MODULE}, 36 | {type, Type} 37 | | options(Type, Opts)]}; 38 | copy_folsom(_, _, _) -> 39 | false. 40 | 41 | new(N, _, Opts) -> 42 | {ok, {proplists:get_value(type, Opts, unknown), 43 | proplists:get_value(folsom_name, Opts, N)}}. 44 | 45 | update(_, Value, counter, {_, Name}) -> 46 | folsom_metrics:notify_existing_metric(Name, {inc,Value}, counter); 47 | update(_, Value, Type, {_, Name}) -> 48 | folsom_metrics:notify_existing_metric(Name, Value, Type). 49 | 50 | reset(_, _, _) -> 51 | {error, unsupported}. 52 | 53 | get_value(_, Type, {_, Name}, DPs) -> 54 | exometer_folsom:get_value(Name, Type, [], DPs). 55 | 56 | sample(_, _, _) -> 57 | {error, unsupported}. 58 | 59 | setopts(_, _, _) -> 60 | ok. 61 | 62 | delete(_, _, _) -> 63 | {error, unsupported}. 64 | 65 | get_datapoints(Name, Type, _) -> 66 | exometer_folsom:get_datapoints(Name, Type, []). 67 | 68 | options(history, [Size]) -> 69 | [{size, Size}]; 70 | options(histogram, [SampleType, SampleSize, Alpha]) -> 71 | [{sample_type, SampleType}, 72 | {sample_size, SampleSize}, 73 | {alpha, Alpha}]; 74 | options(duration , [SampleType, SampleSize, Alpha]) -> 75 | [{sample_type, SampleType}, 76 | {sample_size, SampleSize}, 77 | {alpha, Alpha}]; 78 | options(meter_reader, []) -> []; 79 | options(spiral , []) -> []; 80 | options(meter , []) -> []; 81 | options(gauge , []) -> []; 82 | options(counter , []) -> []. 83 | -------------------------------------------------------------------------------- /src/riak_core_format.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2007-2012 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | %% @doc Functions for formatting data. 22 | 23 | -module(riak_core_format). 24 | -export([fmt/2, 25 | human_size_fmt/2, 26 | human_time_fmt/2, 27 | epoch_to_datetime/1]). 28 | 29 | -ifdef(TEST). 30 | -include_lib("eunit/include/eunit.hrl"). 31 | -endif. 32 | 33 | %% @doc Created a string `Str' based on the format string `FmtStr' and 34 | %% list of args `Args'. 35 | -spec fmt(string(), list()) -> Str::string(). 36 | fmt(FmtStr, Args) -> 37 | lists:flatten(io_lib:format(FmtStr, Args)). 38 | 39 | %% @doc Create a human friendly string `Str' for number of bytes 40 | %% `Bytes' and format based on format string `Fmt'. 41 | -spec human_size_fmt(string(), number()) -> string(). 42 | human_size_fmt(Fmt, Bytes) -> 43 | Fmt2 = Fmt ++ " ~s", 44 | {Value, Units} = human_size(Bytes, ["B","KB","MB","GB","TB","PB"]), 45 | fmt(Fmt2, [Value, Units]). 46 | 47 | %% @doc Create a human friendly string `Str' for the given time in 48 | %% microseconds `Micros'. Format according to format string 49 | %% `Fmt'. 50 | -spec human_time_fmt(string(), non_neg_integer()) -> Str::string(). 51 | human_time_fmt(Fmt, Micros) -> 52 | Fmt2 = Fmt ++ " ~s", 53 | {Value, Units} = human_time(Micros), 54 | fmt(Fmt2, [Value, Units]). 55 | 56 | %% @doc Convert a folsom_utils:now_epoch() to a universal datetime 57 | -spec epoch_to_datetime(non_neg_integer()) -> calendar:datetime(). 58 | epoch_to_datetime(S) -> 59 | Epoch = {{1970,1,1},{0,0,0}}, 60 | Seconds = calendar:datetime_to_gregorian_seconds(Epoch) + S, 61 | calendar:gregorian_seconds_to_datetime(Seconds). 62 | 63 | %%%=================================================================== 64 | %%% Private 65 | %%%=================================================================== 66 | 67 | %% @private 68 | %% 69 | %% @doc Formats a byte size into a human-readable size with units. 70 | %% Thanks StackOverflow: 71 | %% http://stackoverflow.com/questions/2163691/simpler-way-to-format-bytesize-in-a-human-readable-way 72 | -spec human_size(number(), list()) -> {float(), iolist()}. 73 | human_size(S, [_|[_|_] = L]) when S >= 1024 -> human_size(S/1024, L); 74 | human_size(S, [M|_]) -> 75 | {float(S), M}. 76 | 77 | %% @private 78 | %% 79 | %% @doc Given a number of `Micros' returns a human friendly time 80 | %% duration in the form of `{Value, Units}'. 81 | -spec human_time(non_neg_integer()) -> {Value::number(), Units::string()}. 82 | human_time(Micros) -> 83 | human_time(Micros, {1000, "us"}, [{1000, "ms"}, {60, "s"}, {60, "min"}, {24, "hr"}, {365, "d"}]). 84 | 85 | -spec human_time(number(), {pos_integer(), string()}, 86 | [{pos_integer(), string()}]) -> 87 | {float(), string()}. 88 | human_time(T, {Divisor, Unit}, Units) when T < Divisor orelse Units == [] -> 89 | {float(T), Unit}; 90 | human_time(T, {Divisor, _}, [Next|Units]) -> 91 | human_time(T / Divisor, Next, Units). 92 | 93 | -ifdef(TEST). 94 | human_time_fmt_test() -> 95 | FiveUS = 5, 96 | FiveMS = 5000, 97 | FiveS = 5000000, 98 | FiveMin = FiveS * 60, 99 | FiveHr = FiveMin * 60, 100 | FiveDay = FiveHr * 24, 101 | [?assertEqual(Expect, human_time_fmt("~.1f", T)) 102 | || {T, Expect} <- [{FiveUS, "5.0 us"}, 103 | {FiveMS, "5.0 ms"}, 104 | {FiveS, "5.0 s"}, 105 | {FiveMin, "5.0 min"}, 106 | {FiveHr, "5.0 hr"}, 107 | {FiveDay, "5.0 d"}]]. 108 | -endif. 109 | -------------------------------------------------------------------------------- /src/riak_core_handoff_listener.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riak_handoff_listener: entry point for TCP-based handoff 4 | %% 5 | %% Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | %% @doc entry point for TCP-based handoff 24 | 25 | -module(riak_core_handoff_listener). 26 | -behavior(gen_nb_server). 27 | -export([start_link/0]). 28 | -export([init/1, handle_call/3, handle_cast/2, handle_info/2, 29 | terminate/2, code_change/3]). 30 | -export([get_handoff_ip/0, sock_opts/0, new_connection/2]). 31 | -record(state, { 32 | ipaddr :: string(), 33 | portnum :: integer(), 34 | ssl_opts :: list() 35 | }). 36 | 37 | start_link() -> 38 | PortNum = app_helper:get_env(riak_core, handoff_port), 39 | IpAddr = app_helper:get_env(riak_core, handoff_ip), 40 | SslOpts = riak_core_handoff_sender:get_handoff_ssl_options(), 41 | gen_nb_server:start_link(?MODULE, IpAddr, PortNum, [IpAddr, PortNum, SslOpts]). 42 | 43 | get_handoff_ip() -> 44 | riak_core_gen_server:call(?MODULE, handoff_ip, infinity). 45 | 46 | init([IpAddr, PortNum, SslOpts]) -> 47 | register(?MODULE, self()), 48 | 49 | %% This exit() call shouldn't be necessary, AFAICT the VM's EXIT 50 | %% propagation via linking should do the right thing, but BZ 823 51 | %% suggests otherwise. However, the exit() call should fall into 52 | %% the "shouldn't hurt", especially since the next line will 53 | %% explicitly try to spawn a new proc that will try to register 54 | %% the riak_kv_handoff_listener name. 55 | catch exit(whereis(riak_kv_handoff_listener), kill), 56 | process_proxy:start_link(riak_kv_handoff_listener, ?MODULE), 57 | 58 | {ok, #state{portnum=PortNum, ipaddr=IpAddr, ssl_opts = SslOpts}}. 59 | 60 | sock_opts() -> [binary, {packet, 4}, {reuseaddr, true}, {backlog, 64}]. 61 | 62 | handle_call(handoff_ip, _From, State=#state{ipaddr=I}) -> 63 | {reply, {ok, I}, State}; 64 | 65 | handle_call(handoff_port, _From, State=#state{portnum=P}) -> 66 | {reply, {ok, P}, State}. 67 | 68 | handle_cast(_Msg, State) -> {noreply, State}. 69 | 70 | handle_info(_Info, State) -> {noreply, State}. 71 | 72 | terminate(_Reason, _State) -> ok. 73 | 74 | code_change(_OldVsn, State, _Extra) -> {ok, State}. 75 | 76 | new_connection(Socket, State = #state{ssl_opts = SslOpts}) -> 77 | case riak_core_handoff_manager:add_inbound(SslOpts) of 78 | {ok, Pid} -> 79 | ok = gen_tcp:controlling_process(Socket, Pid), 80 | ok = riak_core_handoff_receiver:set_socket(Pid, Socket), 81 | {ok, State}; 82 | {error, _Reason} -> 83 | riak_core_stat:update(rejected_handoffs), 84 | gen_tcp:close(Socket), 85 | {ok, State} 86 | end. 87 | 88 | -------------------------------------------------------------------------------- /src/riak_core_handoff_listener_sup.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2011 Basho Technologies, Inc. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | -module(riak_core_handoff_listener_sup). 22 | -behaviour(supervisor). 23 | 24 | %% beahvior functions 25 | -export([start_link/0, 26 | init/1 27 | ]). 28 | 29 | -define(CHILD(I,Type), {I,{I,start_link,[]},permanent,brutal_kill,Type,[I]}). 30 | 31 | %% begins the supervisor, init/1 will be called 32 | start_link () -> 33 | supervisor:start_link({local,?MODULE},?MODULE,[]). 34 | 35 | %% @private 36 | init ([]) -> 37 | {ok,{{one_for_one,10,10}, 38 | [?CHILD(riak_core_handoff_listener,worker) 39 | ]}}. 40 | -------------------------------------------------------------------------------- /src/riak_core_handoff_receiver_sup.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2011 Basho Technologies, Inc. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | -module(riak_core_handoff_receiver_sup). 22 | -behaviour(supervisor). 23 | 24 | %% beahvior functions 25 | -export([start_link/0, 26 | init/1 27 | ]). 28 | 29 | %% public functions 30 | -export([start_receiver/1 31 | ]). 32 | 33 | -define(CHILD(I,Type), {I,{I,start_link,[]},temporary,brutal_kill,Type,[I]}). 34 | 35 | %% begins the supervisor, init/1 will be called 36 | start_link () -> 37 | supervisor:start_link({local,?MODULE},?MODULE,[]). 38 | 39 | %% @private 40 | init ([]) -> 41 | {ok,{{simple_one_for_one,10,10}, 42 | [?CHILD(riak_core_handoff_receiver,worker) 43 | ]}}. 44 | 45 | %% start a sender process 46 | start_receiver (SSLOpts) -> 47 | supervisor:start_child(?MODULE,[SSLOpts]). 48 | -------------------------------------------------------------------------------- /src/riak_core_handoff_sender_sup.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2011-2012 Basho Technologies, Inc. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | -module(riak_core_handoff_sender_sup). 22 | -behaviour(supervisor). 23 | 24 | %% callbacks 25 | -export([start_link/0, 26 | init/1 27 | ]). 28 | 29 | %% API 30 | -export([start_sender/5]). 31 | 32 | -include("riak_core_handoff.hrl"). 33 | -define(CHILD(I,Type), {I,{I,start_link,[]},temporary,brutal_kill,Type,[I]}). 34 | 35 | %%%=================================================================== 36 | %%% API 37 | %%%=================================================================== 38 | 39 | start_link() -> 40 | supervisor:start_link({local,?MODULE},?MODULE,[]). 41 | 42 | %% @doc Start the handoff process for the module (`Module'), partition 43 | %% (`Partition'), and vnode (`VNode') from the local node to the 44 | %% target node (`TargetNode') with options `Opts'. 45 | %% 46 | %% Options: 47 | %% * src_partition - required. the integer index of the source vnode 48 | %% * target_partition - required. the integer index of the target vnode 49 | %% * filter - optional. an arity one function that takes the key and returns 50 | %% a boolean. If false, the key is not sent 51 | %% * unsent_fun - optional. an arity 2 function that takes a key that was not sent 52 | %% (based on filter) and an accumulator. This function is called 53 | %% for each unsent key. 54 | %% * unsent_acc0 - optional. The intial accumulator value passed to unsent_fun 55 | %% for the first unsent key 56 | -spec start_sender(ho_type(), atom(), term(), pid(), [{atom(), term()}]) -> {ok, pid()}. 57 | start_sender(Type, Module, TargetNode, VNode, Opts) -> 58 | supervisor:start_child(?MODULE, [TargetNode, Module, {Type, Opts}, VNode]). 59 | 60 | %%%=================================================================== 61 | %%% Callbacks 62 | %%%=================================================================== 63 | 64 | %% @private 65 | init ([]) -> 66 | {ok,{{simple_one_for_one,10,10}, 67 | [?CHILD(riak_core_handoff_sender,worker) 68 | ]}}. 69 | -------------------------------------------------------------------------------- /src/riak_core_handoff_sup.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2011 Basho Technologies, Inc. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | -module(riak_core_handoff_sup). 22 | -behaviour(supervisor). 23 | 24 | %% beahvior functions 25 | -export([start_link/0, 26 | init/1 27 | ]). 28 | 29 | -define(CHILD(I,Type), {I,{I,start_link,[]},permanent,brutal_kill,Type,[I]}). 30 | 31 | %% begins the supervisor, init/1 will be called 32 | start_link () -> 33 | supervisor:start_link({local,?MODULE},?MODULE,[]). 34 | 35 | %% @private 36 | init ([]) -> 37 | {ok,{{one_for_all,10,10}, 38 | [?CHILD(riak_core_handoff_receiver_sup,supervisor), 39 | ?CHILD(riak_core_handoff_sender_sup,supervisor), 40 | ?CHILD(riak_core_handoff_listener_sup,supervisor), 41 | ?CHILD(riak_core_handoff_manager,worker) 42 | ]}}. 43 | -------------------------------------------------------------------------------- /src/riak_core_location.erl: -------------------------------------------------------------------------------- 1 | -module(riak_core_location). 2 | 3 | %% API 4 | -export([get_node_location/2, 5 | has_location_set_in_cluster/1, 6 | stripe_nodes_by_location/2, 7 | check_ring/1, 8 | check_ring/2, 9 | check_ring/3, 10 | support_locations_claim/2, 11 | get_location_owners/1, 12 | get_location_nodes/2, 13 | local_nodes/2]). 14 | 15 | -spec get_node_location(node(), dict:dict()) -> string() | undefined. 16 | get_node_location(Node, Locations) -> 17 | case dict:find(Node, Locations) of 18 | error -> 19 | undefined; 20 | {ok, Location} -> 21 | Location 22 | end. 23 | 24 | -spec has_location_set_in_cluster(dict:dict()) -> boolean(). 25 | has_location_set_in_cluster(NodesLocations) -> 26 | 0 =/= dict:size(NodesLocations). 27 | 28 | -spec get_location_nodes([node()|undefined], dict:dict()) -> dict:dict(). 29 | get_location_nodes(Nodes, Locations) -> 30 | lists:foldl(fun(undefined, Acc) -> 31 | dict:append(undefined, undefined, Acc); 32 | (Node, Acc) -> 33 | NodeLocation = get_node_location(Node, Locations), 34 | dict:append(NodeLocation, Node, Acc) 35 | end, dict:new(), Nodes). 36 | 37 | %% Order nodes list by having a different location after each other 38 | -spec stripe_nodes_by_location([node()|undefined], dict:dict()) -> 39 | [node()|undefined]. 40 | stripe_nodes_by_location(Nodes, NodesLocations) -> 41 | LocationsNodes = get_location_nodes(Nodes, NodesLocations), 42 | stripe_nodes_by_location(get_locations(LocationsNodes), LocationsNodes, []). 43 | 44 | -spec get_locations(dict:dict()) -> [node()|undefined]. 45 | get_locations(LocationsNodes) -> 46 | LocationNodesSorted = sort_by_node_count(dict:to_list(LocationsNodes)), 47 | lists:map(fun({Location, _}) -> Location end, LocationNodesSorted). 48 | 49 | -spec sort_by_node_count(list()) -> list(). 50 | sort_by_node_count(LocationsNodes) -> 51 | lists:sort(fun compare/2, LocationsNodes). 52 | 53 | -spec compare({any(), list()}, {any(), list()}) -> boolean(). 54 | compare({_, NodesA}, {_, NodesB}) -> 55 | LengthA = length(NodesA), 56 | LengthB = length(NodesB), 57 | case LengthA =:= LengthB of 58 | true -> 59 | lists:last(NodesA) >= lists:last(NodesB); 60 | _ -> 61 | LengthA >= LengthB 62 | end. 63 | 64 | -spec stripe_nodes_by_location([node()|undefined], dict:dict(), [node()|undefined]) -> 65 | [node()|undefined]. 66 | stripe_nodes_by_location([], _LocationsNodes, NewNodeList) -> 67 | lists:reverse(NewNodeList); 68 | stripe_nodes_by_location(Locations, LocationsNodes, NewNodeList) -> 69 | Nth = (length(NewNodeList) rem length(Locations)) + 1, 70 | CurrentLocation = lists:nth(Nth, Locations), 71 | case dict:find(CurrentLocation, LocationsNodes) of 72 | {ok, []} -> 73 | NewLocations = lists:delete(CurrentLocation, Locations), 74 | NewLocationsNodes = dict:erase(CurrentLocation, LocationsNodes), 75 | stripe_nodes_by_location(NewLocations, NewLocationsNodes, NewNodeList); 76 | {ok, [Node | RemNodes]} -> 77 | NewLocationNodes = dict:store(CurrentLocation, RemNodes, LocationsNodes), 78 | stripe_nodes_by_location(Locations, NewLocationNodes, [Node | NewNodeList]) 79 | end. 80 | 81 | -spec check_ring(riak_core_ring:riak_core_ring()) -> list(). 82 | check_ring(Ring) -> 83 | {ok, Props} = application:get_env(riak_core, default_bucket_props), 84 | check_ring(Ring, proplists:get_value(n_val, Props)). 85 | 86 | -spec check_ring(riak_core_ring:riak_core_ring(), pos_integer()) -> list(). 87 | check_ring(Ring, Nval) -> 88 | check_ring(Ring, Nval, Nval). 89 | 90 | -spec check_ring(riak_core_ring:riak_core_ring(), pos_integer(), pos_integer()) -> list(). 91 | check_ring(Ring, NVal, MinimumNumberOfDistinctLocations) -> 92 | Locations = riak_core_ring:get_nodes_locations(Ring), 93 | case has_location_set_in_cluster(Locations) of 94 | true -> 95 | check_ring(Ring, NVal, MinimumNumberOfDistinctLocations, Locations); 96 | _ -> 97 | [] % no location set, not needed to check 98 | end. 99 | 100 | -spec check_ring(riak_core_ring:riak_core_ring(), pos_integer(), pos_integer(), dict:dict()) -> 101 | list(). 102 | check_ring(Ring, Nval, MinimumNumberOfDistinctLocations, Locations) -> 103 | lists:foldl(fun(PL, Acc) -> 104 | case length(get_unique_locations(PL, Locations)) < MinimumNumberOfDistinctLocations of 105 | false -> Acc; 106 | _ -> 107 | Error = [{Idx, Node, get_node_location(Node, Locations)} || {Idx, Node} <- PL], 108 | [Error | Acc] 109 | end 110 | end, [], riak_core_ring:all_preflists(Ring, Nval)). 111 | 112 | -spec get_unique_locations(list(), dict:dict()) -> 113 | list(). 114 | get_unique_locations(PrefLists, Locations) -> 115 | lists:usort([get_node_location(Node, Locations) || {_, Node} <- PrefLists]). 116 | 117 | %% @doc 118 | %% Are there sufficient locations to support an attempt to cluster claim a 119 | %% given nval. This will be validated before using a location aware claim 120 | %% algorithm 121 | -spec support_locations_claim( 122 | riak_core_ring:riak_core_ring(), pos_integer()) -> boolean(). 123 | support_locations_claim(Ring, TargetNVal) -> 124 | Locations = riak_core_ring:get_nodes_locations(Ring), 125 | case has_location_set_in_cluster(Locations) of 126 | true -> 127 | UniqueLocations = 128 | lists:usort( 129 | lists:map( 130 | fun({_N, L}) -> L end, 131 | dict:to_list(Locations))), 132 | length(UniqueLocations) >= TargetNVal; 133 | false -> 134 | false 135 | end. 136 | 137 | %% @doc 138 | %% Find a mapping between Idx ids and the locations of the nodes which 139 | %% currently own them 140 | -spec get_location_owners( 141 | riak_core_ring:riak_core_ring()) -> list({non_neg_integer(), atom()}). 142 | get_location_owners(Ring) -> 143 | Locations = riak_core_ring:get_nodes_locations(Ring), 144 | lists:map( 145 | fun({Idx, Node}) -> 146 | {Idx, get_node_location(Node, Locations)} 147 | end, 148 | riak_core_ring:all_owners(Ring)). 149 | 150 | %% @doc 151 | %% Find other nodes in the same location of this node 152 | -spec local_nodes(riak_core_ring:riak_core_ring(), node()) -> list(node()). 153 | local_nodes(Ring, Node) -> 154 | Locations = riak_core_ring:get_nodes_locations(Ring), 155 | ThisLocation = get_node_location(Node, Locations), 156 | lists:filtermap( 157 | fun({N, L}) -> 158 | case {N, L} of 159 | {N, ThisLocation} when N =/= Node -> 160 | {true, N}; 161 | _ -> 162 | false 163 | end 164 | end, 165 | dict:to_list(Locations)). -------------------------------------------------------------------------------- /src/riak_core_metadata_object.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2013 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | -module(riak_core_metadata_object). 21 | 22 | -export([value/1, 23 | values/1, 24 | value_count/1, 25 | context/1, 26 | empty_context/0, 27 | hash/1, 28 | modify/4, 29 | reconcile/2, 30 | resolve/2, 31 | is_stale/2, 32 | equal_context/2]). 33 | 34 | -include("riak_core_metadata.hrl"). 35 | 36 | %% @doc returns a single value. if the object holds more than one value an error is generated 37 | %% @see values/2 38 | -spec value(metadata_object()) -> metadata_value(). 39 | value(Metadata) -> 40 | [Value] = values(Metadata), 41 | Value. 42 | 43 | %% @doc returns a list of values held in the object 44 | -spec values(metadata_object()) -> [metadata_value()]. 45 | values({metadata, Object}) -> 46 | [Value || {Value, _Ts} <- dvvset:values(Object)]. 47 | 48 | %% @doc returns the number of siblings in the given object 49 | -spec value_count(metadata_object()) -> non_neg_integer(). 50 | value_count({metadata, Object}) -> 51 | dvvset:size(Object). 52 | 53 | %% @doc returns the context (opaque causal history) for the given object 54 | -spec context(metadata_object()) -> metadata_context(). 55 | context({metadata, Object}) -> 56 | dvvset:join(Object). 57 | 58 | %% @doc returns the representation for an empty context (opaque causal history) 59 | -spec empty_context() -> metadata_context(). 60 | empty_context() -> []. 61 | 62 | %% @doc returns a hash representing the metadata objects contents 63 | -spec hash(metadata_object()) -> binary(). 64 | hash({metadata, Object}) -> 65 | riak_core_util:sha(term_to_binary(Object)). 66 | 67 | %% @doc modifies a potentially existing object, setting its value and updating 68 | %% the causual history. If a function is provided as the third argument 69 | %% then this function also is used for conflict resolution. The difference 70 | %% between this function and resolve/2 is that the logical clock is advanced in the 71 | %% case of this function. Additionally, the resolution functions are slightly different. 72 | -spec modify(metadata_object() | undefined, 73 | metadata_context(), 74 | metadata_value() | metadata_modifier(), 75 | term()) -> metadata_object(). 76 | modify(undefined, Context, Fun, ServerId) when is_function(Fun) -> 77 | modify(undefined, Context, Fun(undefined), ServerId); 78 | modify(Obj, Context, Fun, ServerId) when is_function(Fun) -> 79 | modify(Obj, Context, Fun(values(Obj)), ServerId); 80 | modify(undefined, _Context, Value, ServerId) -> 81 | %% Ignore the context since we dont have a value, its invalid if not 82 | %% empty anyways, so give it a valid one 83 | NewRecord = dvvset:new(timestamped_value(Value)), 84 | {metadata, dvvset:update(NewRecord, ServerId)}; 85 | modify({metadata, Existing}, Context, Value, ServerId) -> 86 | InsertRec = dvvset:new(Context, timestamped_value(Value)), 87 | {metadata, dvvset:update(InsertRec, Existing, ServerId)}. 88 | 89 | %% @doc Reconciles a remote object received during replication or anti-entropy 90 | %% with a local object. If the remote object is an anscestor of or is equal to the local one 91 | %% `false' is returned, otherwise the reconciled object is returned as the second 92 | %% element of the two-tuple 93 | -spec reconcile(metadata_object(), metadata_object() | undefined) -> 94 | false | {true, metadata_object()}. 95 | reconcile(undefined, _LocalObj) -> 96 | false; 97 | reconcile(RemoteObj, undefined) -> 98 | {true, RemoteObj}; 99 | reconcile({metadata, RemoteObj}, {metadata, LocalObj}) -> 100 | Less = dvvset:less(RemoteObj, LocalObj), 101 | Equal = dvvset:equal(RemoteObj, LocalObj), 102 | case not (Equal or Less) of 103 | false -> false; 104 | true -> 105 | {true, {metadata, dvvset:sync([LocalObj, RemoteObj])}} 106 | end. 107 | 108 | %% @doc Resolves siblings using either last-write-wins or the provided function and returns 109 | %% an object containing a single value. The causal history is not updated 110 | -spec resolve(metadata_object(), lww | fun(([metadata_value()]) -> metadata_value())) -> 111 | metadata_object(). 112 | resolve({metadata, Object}, lww) -> 113 | LWW = fun ({_,TS1}, {_,TS2}) -> TS1 =< TS2 end, 114 | {metadata, dvvset:lww(LWW, Object)}; 115 | resolve({metadata, Existing}, Reconcile) when is_function(Reconcile) -> 116 | ResolveFun = fun({A, _}, {B, _}) -> timestamped_value(Reconcile(A, B)) end, 117 | F = fun([Value | Rest]) -> lists:foldl(ResolveFun, Value, Rest) end, 118 | {metadata, dvvset:reconcile(F, Existing)}. 119 | 120 | %% @doc Determines if the given context (version vector) is causually newer than 121 | %% an existing object. If the object missing or if the context does not represent 122 | %% an anscestor of the current key, false is returned. Otherwise, when the context 123 | %% does represent an ancestor of the existing object or the existing object itself, 124 | %% true is returned 125 | -spec is_stale(metadata_context(), metadata_object()) -> boolean(). 126 | is_stale(_, undefined) -> 127 | false; 128 | is_stale(RemoteContext, {metadata, Obj}) -> 129 | LocalContext = dvvset:join(Obj), 130 | %% returns true (stale) when local context is causally newer or equal to remote context 131 | descends(LocalContext, RemoteContext). 132 | 133 | descends(_, []) -> 134 | true; 135 | descends(Ca, Cb) -> 136 | [{NodeB, CtrB} | RestB] = Cb, 137 | case lists:keyfind(NodeB, 1, Ca) of 138 | false -> false; 139 | {_, CtrA} -> 140 | (CtrA >= CtrB) andalso descends(Ca, RestB) 141 | end. 142 | 143 | %% @doc Returns true if the given context and the context of the existing object are equal 144 | -spec equal_context(metadata_context(), metadata_object()) -> boolean(). 145 | equal_context(Context, {metadata, Obj}) -> 146 | Context =:= dvvset:join(Obj). 147 | 148 | timestamped_value(Value) -> 149 | {Value, os:timestamp()}. 150 | -------------------------------------------------------------------------------- /src/riak_core_mochiglobal.erl: -------------------------------------------------------------------------------- 1 | %% @author Bob Ippolito 2 | %% @copyright 2010 Mochi Media, Inc. 3 | %% @doc Abuse module constant pools as a "read-only shared heap" (since erts 5.6) 4 | %% [1]. 5 | -module(riak_core_mochiglobal). 6 | -author("Bob Ippolito "). 7 | -export([get/1, get/2, put/2, delete/1]). 8 | 9 | -spec get(atom()) -> any() | undefined. 10 | %% @equiv get(K, undefined) 11 | get(K) -> 12 | get(K, undefined). 13 | 14 | -spec get(atom(), T) -> any() | T. 15 | %% @doc Get the term for K or return Default. 16 | get(K, Default) -> 17 | get(K, Default, key_to_module(K)). 18 | 19 | get(_K, Default, Mod) -> 20 | try Mod:term() 21 | catch error:undef -> 22 | Default 23 | end. 24 | 25 | -spec put(atom(), any()) -> ok. 26 | %% @doc Store term V at K, replaces an existing term if present. 27 | put(K, V) -> 28 | put(K, V, key_to_module(K)). 29 | 30 | put(_K, V, Mod) -> 31 | Bin = compile(Mod, V), 32 | code:purge(Mod), 33 | {module, Mod} = code:load_binary(Mod, atom_to_list(Mod) ++ ".erl", Bin), 34 | ok. 35 | 36 | -spec delete(atom()) -> boolean(). 37 | %% @doc Delete term stored at K, no-op if non-existent. 38 | delete(K) -> 39 | delete(K, key_to_module(K)). 40 | 41 | delete(_K, Mod) -> 42 | code:purge(Mod), 43 | code:delete(Mod). 44 | 45 | -spec key_to_module(atom()) -> atom(). 46 | key_to_module(K) -> 47 | list_to_atom("mochiglobal:" ++ atom_to_list(K)). 48 | 49 | -spec compile(atom(), any()) -> binary(). 50 | compile(Module, T) -> 51 | {ok, Module, Bin} = compile:forms(forms(Module, T), 52 | [verbose, report_errors]), 53 | Bin. 54 | 55 | -spec forms(atom(), any()) -> [erl_syntax:syntaxTree()]. 56 | forms(Module, T) -> 57 | [erl_syntax:revert(X) || X <- term_to_abstract(Module, term, T)]. 58 | 59 | -spec term_to_abstract(atom(), atom(), any()) -> [erl_syntax:syntaxTree()]. 60 | term_to_abstract(Module, Getter, T) -> 61 | [%% -module(Module). 62 | erl_syntax:attribute( 63 | erl_syntax:atom(module), 64 | [erl_syntax:atom(Module)]), 65 | %% -export([Getter/0]). 66 | erl_syntax:attribute( 67 | erl_syntax:atom(export), 68 | [erl_syntax:list( 69 | [erl_syntax:arity_qualifier( 70 | erl_syntax:atom(Getter), 71 | erl_syntax:integer(0))])]), 72 | %% Getter() -> T. 73 | erl_syntax:function( 74 | erl_syntax:atom(Getter), 75 | [erl_syntax:clause([], none, [erl_syntax:abstract(T)])])]. 76 | 77 | %% 78 | %% Tests 79 | %% 80 | -ifdef(TEST). 81 | -include_lib("eunit/include/eunit.hrl"). 82 | get_put_delete_test() -> 83 | K = '$$test$$mochiglobal', 84 | delete(K), 85 | ?assertEqual( 86 | bar, 87 | get(K, bar)), 88 | try 89 | ?MODULE:put(K, baz), 90 | ?assertEqual( 91 | baz, 92 | get(K, bar)), 93 | ?MODULE:put(K, wibble), 94 | ?assertEqual( 95 | wibble, 96 | ?MODULE:get(K)) 97 | after 98 | delete(K) 99 | end, 100 | ?assertEqual( 101 | bar, 102 | get(K, bar)), 103 | ?assertEqual( 104 | undefined, 105 | ?MODULE:get(K)), 106 | ok. 107 | -endif. 108 | -------------------------------------------------------------------------------- /src/riak_core_net_ticktime.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riak_core: Core Riak Application 4 | %% 5 | %% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | %% @doc Change net_kernel's ticktime on-the-fly. 24 | 25 | -module(riak_core_net_ticktime). 26 | -export([enable/0, 27 | start_set_net_ticktime_daemon/2, 28 | stop_set_net_ticktime_daemon/1]). 29 | 30 | -include_lib("kernel/include/logger.hrl"). 31 | 32 | -define(REGNAME, net_kernel_net_ticktime_change_daemon). 33 | 34 | -spec enable() -> ok. 35 | enable() -> 36 | riak_core_capability:register({riak_core, net_ticktime}, 37 | [true, false], 38 | false). 39 | 40 | start_set_net_ticktime_daemon(Node, Time) -> 41 | start_set_net_ticktime_daemon(Node, Time, net_ticktime_active()). 42 | 43 | start_set_net_ticktime_daemon(Node, Time, true) -> 44 | EbinDir = filename:dirname(code:which(?MODULE)), 45 | try 46 | Dirs = rpc:call(Node, code, get_path, []), 47 | case lists:member(EbinDir, Dirs) of 48 | false -> 49 | ?LOG_INFO("start_set_net_ticktime_daemon: adding to code path " 50 | "for node ~p\n", [Node]), 51 | rpc:call(Node, code, add_pathz, [EbinDir]); 52 | true -> 53 | ok 54 | end 55 | catch _:_ -> 56 | %% Network problems or timeouts here, spawn will fail, no 57 | %% worries, we'll try again soon. 58 | ok 59 | end, 60 | spawn(Node, fun() -> 61 | try 62 | register(?REGNAME, self()), 63 | %% If we get here, we are the one daemon process 64 | ?LOG_INFO("start_set_net_ticktime_daemon: started " 65 | "changing net_ticktime on ~p to ~p\n", 66 | [Node, Time]), 67 | _ = rand:seed(exs64, os:timestamp()), 68 | set_net_ticktime_daemon_loop(Time, 1) 69 | catch _:_ -> 70 | ok 71 | end 72 | end); 73 | start_set_net_ticktime_daemon(Node, _Time, false) -> 74 | ?LOG_INFO("Not starting tick daemon on ~p. Capability unsupported. " 75 | "Some nodes in the Riak cluster do not have ~p loaded\n", 76 | [Node, ?MODULE]), 77 | ok. 78 | 79 | stop_set_net_ticktime_daemon(Node) -> 80 | Capability = riak_core_capability:get({riak_core, net_ticktime}), 81 | stop_set_net_ticktime_daemon(Node, Capability). 82 | 83 | stop_set_net_ticktime_daemon(Node, true) -> 84 | try 85 | case rpc:call(Node, erlang, whereis, [?REGNAME]) of 86 | Pid when is_pid(Pid) -> 87 | io:format("Stopping tick daemon ~p on ~p\n", [Pid, Node]), 88 | exit(Pid, stop_now), 89 | ok; 90 | undefined -> 91 | io:format("Stopping tick daemon on ~p but not running\n", [Node]), 92 | ok 93 | end 94 | catch _:_ -> 95 | %% Network problems or timeouts, we don't try too hard 96 | error 97 | end; 98 | stop_set_net_ticktime_daemon(Node, false) -> 99 | ?LOG_INFO("Not stopping tick daemon on ~p. Capability unsupported\n", [Node]), 100 | ok. 101 | 102 | async_start_set_net_ticktime_daemons(Time, Nodes) -> 103 | Pids = [spawn(fun() -> 104 | start_set_net_ticktime_daemon(Node, Time, true) 105 | end) || Node <- Nodes], 106 | spawn(fun() -> 107 | %% If a daemon cannot finish in 5 seconds, no worries. 108 | %% We want to avoid leaving lots of pids around due to 109 | %% network/net_kernel instability. 110 | timer:sleep(5000), 111 | [exit(Pid, kill) || Pid <- Pids] 112 | end). 113 | 114 | set_net_ticktime_daemon_loop(Time, Count) -> 115 | case set_net_ticktime(Time) of 116 | unchanged -> 117 | ?LOG_INFO("start_set_net_ticktime_daemon: finished " 118 | "changing net_ticktime on ~p to ~p\n", [node(), Time]), 119 | exit(normal); 120 | _ -> 121 | timer:sleep(rand:uniform(1*1000)), 122 | %% Here is an uncommon use the erlang:nodes/1 BIF. 123 | %% Hidden nodes (e.g. administrative escripts) may have 124 | %% connected to us. Force them to change their tick time, 125 | %% in case they're using something different. And pick up 126 | %% any regular nodes that have connected since we started. 127 | if 128 | Count rem 5 == 0 -> 129 | async_start_set_net_ticktime_daemons( 130 | Time, da_nodes(nodes(connected))), 131 | ok; 132 | true -> 133 | ok 134 | end, 135 | set_net_ticktime_daemon_loop(Time, Count + 1) 136 | end. 137 | 138 | da_nodes(Nodes) -> 139 | lists:sort([node()|Nodes]). % Always include myself 140 | 141 | set_net_ticktime(Time) -> 142 | case net_kernel:set_net_ticktime(Time) of 143 | {Status, _} -> 144 | Status; 145 | A when is_atom(A) -> 146 | A 147 | end. 148 | 149 | -spec net_ticktime_active() -> boolean(). 150 | net_ticktime_active() -> 151 | case catch riak_core_capability:get({riak_core, net_ticktime}) of 152 | true -> 153 | true; 154 | _ -> 155 | false 156 | end. 157 | -------------------------------------------------------------------------------- /src/riak_core_node_watcher_events.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riak_core: Core Riak Application 4 | %% 5 | %% Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | -module(riak_core_node_watcher_events). 23 | 24 | -behaviour(gen_event). 25 | 26 | %% API 27 | -export([start_link/0, 28 | add_handler/2, 29 | add_sup_handler/2, 30 | add_guarded_handler/2, 31 | add_callback/1, 32 | add_sup_callback/1, 33 | add_guarded_callback/1, 34 | service_update/1]). 35 | 36 | %% gen_event callbacks 37 | -export([init/1, handle_event/2, handle_call/2, 38 | handle_info/2, terminate/2, code_change/3]). 39 | 40 | -record(state, { callback }). 41 | 42 | %% =================================================================== 43 | %% API functions 44 | %% =================================================================== 45 | 46 | start_link() -> 47 | gen_event:start_link({local, ?MODULE}). 48 | 49 | add_handler(Handler, Args) -> 50 | gen_event:add_handler(?MODULE, Handler, Args). 51 | 52 | add_sup_handler(Handler, Args) -> 53 | gen_event:add_sup_handler(?MODULE, Handler, Args). 54 | 55 | add_guarded_handler(Handler, Args) -> 56 | riak_core:add_guarded_event_handler(?MODULE, Handler, Args). 57 | 58 | add_callback(Fn) when is_function(Fn) -> 59 | gen_event:add_handler(?MODULE, {?MODULE, make_ref()}, [Fn]). 60 | 61 | add_sup_callback(Fn) when is_function(Fn) -> 62 | gen_event:add_sup_handler(?MODULE, {?MODULE, make_ref()}, [Fn]). 63 | 64 | add_guarded_callback(Fn) when is_function(Fn) -> 65 | riak_core:add_guarded_event_handler(?MODULE, {?MODULE, make_ref()}, [Fn]). 66 | 67 | service_update(Services) -> 68 | gen_event:notify(?MODULE, {service_update, Services}). 69 | 70 | 71 | %% =================================================================== 72 | %% gen_event callbacks 73 | %% =================================================================== 74 | 75 | init([Fn]) -> 76 | %% Get the initial list of available services 77 | Fn(riak_core_node_watcher:services()), 78 | {ok, #state { callback = Fn }}. 79 | 80 | handle_event({service_update, Services}, State) -> 81 | (State#state.callback)(Services), 82 | {ok, State}. 83 | 84 | handle_call(_Request, State) -> 85 | {ok, ok, State}. 86 | 87 | handle_info(_Info, State) -> 88 | {ok, State}. 89 | 90 | terminate(_Reason, _State) -> 91 | ok. 92 | 93 | code_change(_OldVsn, State, _Extra) -> 94 | {ok, State}. 95 | 96 | -------------------------------------------------------------------------------- /src/riak_core_node_worker_pool.erl: -------------------------------------------------------------------------------- 1 | %% 2 | %% Copyright (c) 2007-2011 Basho Technologies, Inc. All Rights Reserved. 3 | %% 4 | %% This file is provided to you under the Apache License, 5 | %% Version 2.0 (the "License"); you may not use this file 6 | %% except in compliance with the License. You may obtain 7 | %% a copy of the License at 8 | %% 9 | %% http://www.apache.org/licenses/LICENSE-2.0 10 | %% 11 | %% Unless required by applicable law or agreed to in writing, 12 | %% software distributed under the License is distributed on an 13 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | %% KIND, either express or implied. See the License for the 15 | %% specific language governing permissions and limitations 16 | %% under the License. 17 | %% 18 | %% ------------------------------------------------------------------- 19 | 20 | -module(riak_core_node_worker_pool). 21 | 22 | -behaviour(riak_core_worker_pool). 23 | 24 | -export([do_init/1, reply/2, do_work/3, to_log/0]). 25 | %% export the names of the pools as functions 26 | -export([af1/0, af2/0, af3/0, af4/0, be/0, nwp/0, dscp_pools/0, pools/0]). 27 | 28 | %% API 29 | -export([start_link/5, stop/2, shutdown_pool/2, handle_work/3]). 30 | 31 | -include_lib("kernel/include/logger.hrl"). 32 | 33 | -type worker_pool() 34 | % Allows you to set up a DSCP-style set of pools (assuming the 35 | % vnode_worker_pool counts as ef. Otherwise can just have a 36 | % single node_worker_pool 37 | :: be_pool|af1_pool|af2_pool|af3_pool|af4_pool|node_worker_pool. 38 | 39 | -export_type([worker_pool/0]). 40 | 41 | -spec af1() -> af1_pool. 42 | af1() -> af1_pool. 43 | 44 | -spec af2() -> af2_pool. 45 | af2() -> af2_pool. 46 | 47 | -spec af3() -> af3_pool. 48 | af3() -> af3_pool. 49 | 50 | -spec af4() -> af4_pool. 51 | af4() -> af4_pool. 52 | 53 | -spec be() -> be_pool. 54 | be() -> be_pool. 55 | 56 | -spec nwp() -> node_worker_pool. 57 | nwp() -> node_worker_pool. 58 | 59 | -spec pools() -> [worker_pool()]. 60 | pools() -> 61 | [af1(), af2(), af3(), af4(), be(), nwp()]. 62 | 63 | -spec dscp_pools() -> [worker_pool()]. 64 | dscp_pools() -> 65 | [af1(), af2(), af3(), af4(), be()]. 66 | 67 | -spec start_link(atom(), pos_integer(), list(), list(), worker_pool()) 68 | -> {ok, pid()}. 69 | %% @doc 70 | %% Start a worker pool, and register under the name PoolType, which should be 71 | %% a recognised name from type worker_pool() 72 | start_link(WorkerMod, PoolSize, WorkerArgs, WorkerProps, PoolType) 73 | when PoolType == be_pool; 74 | PoolType == af1_pool; 75 | PoolType == af2_pool; 76 | PoolType == af3_pool; 77 | PoolType == af4_pool; 78 | PoolType == node_worker_pool -> 79 | {ok, Pid} = 80 | riak_core_worker_pool:start_link( 81 | [WorkerMod, PoolSize, WorkerArgs, WorkerProps], 82 | ?MODULE, 83 | PoolType), 84 | register(PoolType, Pid), 85 | ?LOG_INFO("Registered worker pool of type ~w and size ~w", 86 | [PoolType, PoolSize]), 87 | {ok, Pid}. 88 | 89 | do_init([WorkerMod, PoolSize, WorkerArgs, WorkerProps]) -> 90 | process_flag(trap_exit, true), 91 | poolboy:start_link([{worker_module, riak_core_vnode_worker}, 92 | {worker_args, 93 | [node, WorkerArgs, WorkerProps, self()]}, 94 | {worker_callback_mod, WorkerMod}, 95 | {size, PoolSize}, {max_overflow, 0}]). 96 | 97 | handle_work(PoolName, Work, From) when 98 | PoolName == be_pool; 99 | PoolName == af1_pool; 100 | PoolName == af2_pool; 101 | PoolName == af3_pool; 102 | PoolName == af4_pool; 103 | PoolName == node_worker_pool -> 104 | riak_core_stat:update({worker_pool, PoolName}), 105 | riak_core_worker_pool:handle_work(PoolName, Work, From). 106 | 107 | stop(Pid, Reason) -> 108 | riak_core_worker_pool:stop(Pid, Reason). 109 | 110 | %% wait for all the workers to finish any current work 111 | shutdown_pool(Pid, Wait) -> 112 | riak_core_worker_pool:shutdown_pool(Pid, Wait). 113 | 114 | reply(From, Msg) -> 115 | riak_core_vnode:reply(From, Msg). 116 | 117 | do_work(Pid, Work, From) -> 118 | riak_core_vnode_worker:handle_work(Pid, Work, From). 119 | 120 | to_log() -> true. 121 | -------------------------------------------------------------------------------- /src/riak_core_node_worker_pool_sup.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% This file is provided to you under the Apache License, 4 | %% Version 2.0 (the "License"); you may not use this file 5 | %% except in compliance with the License. You may obtain 6 | %% a copy of the License at 7 | %% 8 | %% http://www.apache.org/licenses/LICENSE-2.0 9 | %% 10 | %% Unless required by applicable law or agreed to in writing, 11 | %% software distributed under the License is distributed on an 12 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 13 | %% KIND, either express or implied. See the License for the 14 | %% specific language governing permissions and limitations 15 | %% under the License. 16 | %% 17 | %% ------------------------------------------------------------------- 18 | -module(riak_core_node_worker_pool_sup). 19 | -behaviour(supervisor). 20 | -export([start_link/0, init/1]). 21 | -export([start_pool/5]). 22 | 23 | %% Helper macro for declaring children of supervisor 24 | -define(CHILD(I, PoolType, Args, Type, Timeout), 25 | {PoolType, 26 | {I, start_link, Args}, 27 | permanent, Timeout, Type, [I]}). 28 | -define(CHILD(I, PoolType, Args, Type), 29 | ?CHILD(I, PoolType, Args, Type, 5000)). 30 | 31 | -type worker_pool() :: riak_core_node_worker_pool:worker_pool(). 32 | 33 | start_link() -> 34 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 35 | 36 | init([]) -> 37 | {ok, {{one_for_one, 5, 10}, []}}. 38 | 39 | %% @doc 40 | %% Start a node_worker_pool - can be either assuredforwardng_pool or 41 | %% a besteffort_pool (which will also be registered as a node_worker_pool for 42 | %% backwards compatability) 43 | -spec start_pool(atom(), pos_integer(), list(), list(), worker_pool()) -> 44 | ok | {error, Reason::term()}. 45 | start_pool(WorkerMod, PoolSize, WorkerArgs, WorkerProps, QueueType) -> 46 | Ref = pool(WorkerMod, PoolSize, WorkerArgs, WorkerProps, QueueType), 47 | case supervisor:start_child(?MODULE, Ref) of 48 | {ok, _} -> ok; 49 | {ok, _, _} -> ok; 50 | {error, already_present} -> ok; 51 | {error, {already_started, _}} -> ok; 52 | {error, OtherErr} -> {error, OtherErr} 53 | end. 54 | 55 | pool(WorkerMod, PoolSize, WorkerArgs, WorkerProps, QueueType) -> 56 | ?CHILD(riak_core_node_worker_pool, 57 | QueueType, 58 | [WorkerMod, PoolSize, WorkerArgs, WorkerProps, QueueType], 59 | worker). 60 | 61 | -------------------------------------------------------------------------------- /src/riak_core_nodeid.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riak_core: Core Node Id 4 | %% 5 | %% Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | %% Return a binary to use as an identifier for this node. 23 | %% Initially this is just hash the node name, so there is a small 24 | %% chance of collisions. 25 | %% ------------------------------------------------------------------- 26 | -module(riak_core_nodeid). 27 | -export([get/0]). 28 | 29 | get() -> 30 | Id = erlang:crc32(term_to_binary(node())), 31 | <>. 32 | -------------------------------------------------------------------------------- /src/riak_core_pw_auth.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2013 Basho Technologies, Inc. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | -module(riak_core_pw_auth). 22 | 23 | -export([hash_password/1, check_password/5]). 24 | 25 | %% TOOD should make these configurable in app.config 26 | -define(SALT_LENGTH, 16). 27 | -define(HASH_ITERATIONS, 65536). 28 | %% TODO this should call a default_hash_func() function to get default based on erlang version 29 | -define(HASH_FUNCTION, sha). 30 | -define(AUTH_NAME, pbkdf2). 31 | 32 | %% @doc Hash a plaintext password, returning hashed password and algorithm details 33 | hash_password(BinaryPass) when is_binary(BinaryPass) -> 34 | % TODO: Do something more with the salt? 35 | % Generate salt the simple way 36 | Salt = crypto:strong_rand_bytes(?SALT_LENGTH), 37 | 38 | % Hash the original password and store as hex 39 | {ok, HashedPass} = pbkdf2:pbkdf2(?HASH_FUNCTION, BinaryPass, Salt, ?HASH_ITERATIONS), 40 | HexPass = pbkdf2:to_hex(HashedPass), 41 | {ok, HexPass, ?AUTH_NAME, ?HASH_FUNCTION, Salt, ?HASH_ITERATIONS}. 42 | 43 | 44 | %% @doc Check a plaintext password with a hashed password 45 | check_password(BinaryPass, HashedPassword, HashFunction, Salt, HashIterations) when is_binary(BinaryPass) -> 46 | 47 | % Hash EnteredPassword to compare to HashedPassword 48 | {ok, HashedPass} = pbkdf2:pbkdf2(HashFunction, BinaryPass, Salt, HashIterations), 49 | HexPass = pbkdf2:to_hex(HashedPass), 50 | pbkdf2:compare_secure(HexPass, HashedPassword). 51 | -------------------------------------------------------------------------------- /src/riak_core_repair.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2012 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | -module(riak_core_repair). 22 | -export([gen_filter/5, 23 | gen_range/3, 24 | gen_range_fun/2, 25 | gen_range_map/3]). 26 | 27 | -include("riak_core_vnode.hrl"). 28 | 29 | %% GTE = Greater Than or Equal 30 | %% LTE = Less Than or Equal 31 | -type range_wrap() :: {wrap, GTE::binary(), LTE::binary()}. 32 | -type range_nowrap() :: {nowrap, GTE::binary(), LTE::binary()}. 33 | -type hash_range() :: range_wrap() | range_nowrap(). 34 | 35 | %% =================================================================== 36 | %% Public API 37 | %% =================================================================== 38 | 39 | %% @doc Generate a `Filter' fun to use during partition repair. 40 | %% 41 | %% `Target' - Partition under repair. 42 | %% 43 | %% `Ring' - The ring to use for repair. 44 | %% 45 | %% `NValMap' - A map from bucket to `n_val', only custom buckets 46 | %% have entries, everything else uses default. 47 | %% 48 | %% `DefaultN' - The default `n_val'. 49 | %% 50 | %% `InfoFun' - A function which returns information about the key 51 | %% used to determine if it should be repaired or not. 52 | gen_filter(Target, Ring, NValMap, DefaultN, InfoFun) -> 53 | RangeMap = riak_core_repair:gen_range_map(Target, Ring, NValMap), 54 | Default = riak_core_repair:gen_range(Target, Ring, DefaultN), 55 | RangeFun = riak_core_repair:gen_range_fun(RangeMap, Default), 56 | fun(BKey) -> 57 | {Bucket, Hash} = InfoFun(BKey), 58 | case RangeFun(Bucket) of 59 | {nowrap, GTE, LTE} -> 60 | Hash >= GTE andalso Hash =< LTE; 61 | {wrap, GTE, LTE} -> 62 | Hash >= GTE orelse Hash =< LTE 63 | end 64 | end. 65 | 66 | %% @doc Generate the hash `Range' for a given `Target' partition and 67 | %% `NVal'. 68 | %% 69 | %% Note: The type of NVal should be pos_integer() but dialyzer says 70 | %% success typing is integer() and I don't have time for games. 71 | -spec gen_range(partition(), riak_core_ring:riak_core_ring(), integer()) -> 72 | hash_range(). 73 | gen_range(Target, Ring, NVal) -> 74 | CH = riak_core_ring:chash(Ring), 75 | Predecessors = chash:predecessors(<>, CH, NVal+1), 76 | [{FirstIdx, Node}|Rest] = Predecessors, 77 | Predecessors2 = [{FirstIdx-1, Node}|Rest], 78 | Predecessors3 = [<> || {I,_} <- Predecessors2], 79 | {A,B} = lists:splitwith(fun(E) -> E > <<0:160/integer>> end, Predecessors3), 80 | case B of 81 | [] -> 82 | %% In this case there is no "wrap" around the end of the 83 | %% ring so the range check it simply an inclusive 84 | %% inbetween. 85 | make_nowrap(A); 86 | 87 | [<<0:160/integer>>] -> 88 | %% In this case the 0th partition is first so no wrap 89 | %% actually occurred. 90 | make_nowrap(A ++ B); 91 | 92 | _ -> 93 | %% In this case there is a "wrap" around the end of the 94 | %% ring. Either the key is greater than or equal the 95 | %% largest or smaller than or equal to the smallest. 96 | make_wrap(A, B) 97 | end. 98 | 99 | 100 | %% @doc Generate the function that will return the hash range for a 101 | %% given `Bucket'. 102 | -spec gen_range_fun(list(), hash_range()) -> function(). 103 | gen_range_fun(RangeMap, Default) -> 104 | fun(Bucket) -> 105 | case lists:keyfind(Bucket, 1, RangeMap) of 106 | false -> Default; 107 | {_, Val} -> Val 108 | end 109 | end. 110 | 111 | %% @doc Generate the map from bucket `B' to hash `Range' that a key 112 | %% must fall into to be included for repair on the `Target' 113 | %% partition. 114 | -spec gen_range_map(partition(), riak_core_ring:riak_core_ring(), list()) -> 115 | [{Bucket::binary(), Range::hash_range()}]. 116 | gen_range_map(Target, Ring, NValMap) -> 117 | [{I, gen_range(Target, Ring, N)} || {I, N} <- NValMap, N > 1]. 118 | 119 | 120 | %% =================================================================== 121 | %% Internal 122 | %% =================================================================== 123 | 124 | %% @private 125 | %% 126 | %% @doc Make the `nowrap' tuple representing the range a key hash must 127 | %% fall into. 128 | -spec make_nowrap(list()) -> range_nowrap(). 129 | make_nowrap(RingSlice) -> 130 | Slice2 = lists:sort(RingSlice), 131 | GTE = hd(Slice2), 132 | LTE = lists:last(Slice2), 133 | {nowrap, GTE, LTE}. 134 | 135 | %% @private 136 | %% 137 | %% @doc Make `wrap' tuple representing the two ranges, relative to 0, 138 | %% that a key hash must fall into. 139 | %% 140 | %% `RingSliceAfter' contains entries after the wrap around the 0th 141 | %% partition. `RingSliceBefore' are entries before wrap and the 142 | %% including the wrap at partition 0. 143 | -spec make_wrap(list(), list()) -> range_wrap(). 144 | make_wrap(RingSliceAfter, RingSliceBefore) -> 145 | LTE = hd(RingSliceAfter), 146 | %% know first element of RingSliceBefore is 0 147 | Before2 = tl(RingSliceBefore), 148 | GTE = lists:last(Before2), 149 | {wrap, GTE, LTE}. 150 | 151 | -------------------------------------------------------------------------------- /src/riak_core_ring_events.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riak_core: Core Riak Application 4 | %% 5 | %% Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | -module(riak_core_ring_events). 23 | 24 | -behaviour(gen_event). 25 | 26 | %% API 27 | -export([start_link/0, 28 | add_handler/2, 29 | add_sup_handler/2, 30 | add_guarded_handler/2, 31 | add_callback/1, 32 | add_sup_callback/1, 33 | add_guarded_callback/1, 34 | ring_update/1, 35 | force_update/0, 36 | ring_sync_update/1, 37 | force_sync_update/0]). 38 | 39 | %% gen_event callbacks 40 | -export([init/1, handle_event/2, handle_call/2, 41 | handle_info/2, terminate/2, code_change/3]). 42 | 43 | -record(state, { callback }). 44 | 45 | %% =================================================================== 46 | %% API functions 47 | %% =================================================================== 48 | 49 | start_link() -> 50 | gen_event:start_link({local, ?MODULE}). 51 | 52 | add_handler(Handler, Args) -> 53 | gen_event:add_handler(?MODULE, Handler, Args). 54 | 55 | add_sup_handler(Handler, Args) -> 56 | gen_event:add_sup_handler(?MODULE, Handler, Args). 57 | 58 | add_guarded_handler(Handler, Args) -> 59 | riak_core:add_guarded_event_handler(?MODULE, Handler, Args). 60 | 61 | add_callback(Fn) when is_function(Fn) -> 62 | gen_event:add_handler(?MODULE, {?MODULE, make_ref()}, [Fn]). 63 | 64 | add_sup_callback(Fn) when is_function(Fn) -> 65 | gen_event:add_sup_handler(?MODULE, {?MODULE, make_ref()}, [Fn]). 66 | 67 | add_guarded_callback(Fn) when is_function(Fn) -> 68 | riak_core:add_guarded_event_handler(?MODULE, {?MODULE, make_ref()}, [Fn]). 69 | 70 | force_update() -> 71 | {ok, Ring} = riak_core_ring_manager:get_my_ring(), 72 | ring_update(Ring). 73 | 74 | ring_update(Ring) -> 75 | gen_event:notify(?MODULE, {ring_update, Ring}). 76 | 77 | force_sync_update() -> 78 | {ok, Ring} = riak_core_ring_manager:get_my_ring(), 79 | ring_sync_update(Ring). 80 | 81 | ring_sync_update(Ring) -> 82 | gen_event:sync_notify(?MODULE, {ring_update, Ring}). 83 | 84 | %% =================================================================== 85 | %% gen_event callbacks 86 | %% =================================================================== 87 | 88 | init([Fn]) -> 89 | {ok, Ring} = riak_core_ring_manager:get_my_ring(), 90 | Fn(Ring), 91 | {ok, #state { callback = Fn }}. 92 | 93 | handle_event({ring_update, Ring}, State) -> 94 | (State#state.callback)(Ring), 95 | {ok, State}. 96 | 97 | handle_call(_Request, State) -> 98 | {ok, ok, State}. 99 | 100 | handle_info(_Info, State) -> 101 | {ok, State}. 102 | 103 | terminate(_Reason, _State) -> 104 | ok. 105 | 106 | code_change(_OldVsn, State, _Extra) -> 107 | {ok, State}. 108 | 109 | -------------------------------------------------------------------------------- /src/riak_core_send_msg.erl: -------------------------------------------------------------------------------- 1 | %% 2 | %% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. 3 | %% 4 | %% This file is provided to you under the Apache License, 5 | %% Version 2.0 (the "License"); you may not use this file 6 | %% except in compliance with the License. You may obtain 7 | %% a copy of the License at 8 | %% 9 | %% http://www.apache.org/licenses/LICENSE-2.0 10 | %% 11 | %% Unless required by applicable law or agreed to in writing, 12 | %% software distributed under the License is distributed on an 13 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | %% KIND, either express or implied. See the License for the 15 | %% specific language governing permissions and limitations 16 | %% under the License. 17 | %% 18 | %% ------------------------------------------------------------------- 19 | 20 | %% @doc OTP equivalents for sending reply- and event-like things 21 | %% without blocking the sender. 22 | 23 | -module(riak_core_send_msg). 24 | 25 | -export([reply_unreliable/2, 26 | cast_unreliable/2, 27 | send_event_unreliable/2, 28 | bang_unreliable/2]). 29 | 30 | -ifdef(TEST). 31 | -ifdef(PULSE). 32 | -compile(export_all). 33 | -compile({parse_transform, pulse_instrument}). 34 | -compile({pulse_replace_module, [{gen_server, pulse_gen_server}]}). 35 | -endif. 36 | -endif. 37 | 38 | %% NOTE: We'ed peeked inside gen_server.erl's guts to see its internals. 39 | reply_unreliable({To, Tag}, Reply) -> 40 | bang_unreliable(To, {Tag, Reply}). 41 | 42 | cast_unreliable(Dest, Request) -> 43 | bang_unreliable(Dest, {'$gen_cast', Request}). 44 | 45 | %% NOTE: We'ed peeked inside gen_fsm_compat.erl's guts to see its internals. 46 | send_event_unreliable({global, _Name} = GlobalTo, Event) -> 47 | erlang:error({unimplemented_send, GlobalTo, Event}); 48 | send_event_unreliable({via, _Mod, _Name} = ViaTo, Event) -> 49 | erlang:error({unimplemented_send, ViaTo, Event}); 50 | send_event_unreliable(Name, Event) -> 51 | bang_unreliable(Name, {'$gen_event', Event}), 52 | ok. 53 | 54 | bang_unreliable(Dest, Msg) -> 55 | catch erlang:send(Dest, Msg, [noconnect, nosuspend]), 56 | Msg. 57 | -------------------------------------------------------------------------------- /src/riak_core_stat_cache.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | -module(riak_core_stat_cache). 21 | 22 | -behaviour(gen_server). 23 | 24 | -export([start_link/0, 25 | stop/0]). 26 | 27 | -export([register_app/2, 28 | get_stats/1]). 29 | 30 | -export([init/1, 31 | handle_call/3, 32 | handle_cast/2, 33 | handle_info/2, 34 | terminate/2, 35 | code_change/3]). 36 | 37 | -record(st, {apps = []}). 38 | 39 | register_app(App, ProduceMFA) -> 40 | gen_server:call(?MODULE, {register, App, ProduceMFA}), 41 | ok. 42 | 43 | %% This function should not be called by code that's been adapted to 44 | %% exometer. Thus, it must return the old format, and assume that the 45 | %% 46 | get_stats(App) -> 47 | T1 = os:timestamp(), 48 | Result = case get_app(App) of 49 | {_, {M, F, A}} -> 50 | apply(M, F, A); 51 | false -> 52 | [] 53 | end, 54 | T2 = os:timestamp(), 55 | {ok, Result, timer:now_diff(T2, T1)}. 56 | 57 | %% The gen_server mainly exists as a stub, since legacy test suites expect 58 | %% riak_core_stat_cache:start_link() to return {ok, Pid}. It also keeps track 59 | %% of the produce_stats() callbacks for each registered app. 60 | 61 | start_link() -> 62 | gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). 63 | 64 | stop() -> 65 | ok. 66 | 67 | init(_) -> {ok, #st{}}. 68 | 69 | handle_call({register, App, MFA}, _, #st{apps = Apps} = S) -> 70 | {reply, ok, S#st{apps = lists:keystore(App, 1, Apps, {App, MFA})}}; 71 | handle_call({get_app, App}, _, #st{apps = Apps} = S) -> 72 | {reply, lists:keyfind(App, 1, Apps), S}; 73 | handle_call(_, _, S) -> 74 | {reply, {error,not_supported}, S}. 75 | 76 | handle_cast(_, S) -> {noreply, S}. 77 | handle_info(_, S) -> {noreply, S}. 78 | terminate(_, _) -> ok. 79 | code_change(_, S, _) -> {ok, S}. 80 | 81 | %% -- end gen_server 82 | 83 | get_app(App) -> 84 | gen_server:call(?MODULE, {get_app, App}). 85 | -------------------------------------------------------------------------------- /src/riak_core_stat_calc_sup.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% Copyright (c) 2007-2011 Basho Technologies, Inc. All Rights Reserved. 3 | %% 4 | %% This file is provided to you under the Apache License, 5 | %% Version 2.0 (the "License"); you may not use this file 6 | %% except in compliance with the License. You may obtain 7 | %% a copy of the License at 8 | %% 9 | %% http://www.apache.org/licenses/LICENSE-2.0 10 | %% 11 | %% Unless required by applicable law or agreed to in writing, 12 | %% software distributed under the License is distributed on an 13 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | %% KIND, either express or implied. See the License for the 15 | %% specific language governing permissions and limitations 16 | %% under the License. 17 | %% 18 | %% ------------------------------------------------------------------- 19 | -module(riak_core_stat_calc_sup). 20 | -behaviour(supervisor). 21 | -export([start_link/0, init/1]). 22 | -export([calc_proc/1, stop_proc/1]). 23 | 24 | %% Helper macro for declaring children of supervisor 25 | -define(CHILD(I, Type, Timeout), {I, {riak_core_stat_calc_proc, start_link, [I]}, permanent, Timeout, Type, [riak_core_stat_calc_proc]}). 26 | -define(CHILD(I, Type), ?CHILD(I, Type, 5000)). 27 | 28 | start_link() -> 29 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 30 | 31 | init([]) -> 32 | {ok, {{one_for_one, 5, 10}, []}}. 33 | 34 | %% start a process for the given stat 35 | %% {@see riak_core_stat_calc_proc} 36 | calc_proc(Stat) -> 37 | Ref = calc_proc_ref(Stat), 38 | Pid = case supervisor:start_child(?MODULE, Ref) of 39 | {ok, Child} -> Child; 40 | {error, {already_started, Child}} -> Child 41 | end, 42 | Pid. 43 | 44 | stop_proc(Stat) -> 45 | _ = supervisor:terminate_child(?MODULE, Stat), 46 | _ = supervisor:delete_child(?MODULE, Stat), 47 | ok. 48 | 49 | %% @private 50 | calc_proc_ref(Stat) -> 51 | ?CHILD(Stat, worker). 52 | -------------------------------------------------------------------------------- /src/riak_core_stat_q.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2007-2012 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | %% @doc riak_core_stat_q is an interface to query folsom stats 22 | %% To use, call `get_stats/1' with a query `Path'. 23 | %% A `Path' is a list of atoms | binaries. The module creates a set 24 | %% of `ets:select/1' guards, one for each element in `Path' 25 | %% For each stat that has a key that matches `Path' we calculate the 26 | %% current value and return it. 27 | 28 | -module(riak_core_stat_q). 29 | 30 | -compile(nowarn_export_all). 31 | -compile(export_all). 32 | 33 | -export_type([path/0, 34 | stat_name/0]). 35 | 36 | -include_lib("kernel/include/logger.hrl"). 37 | 38 | -type path() :: [] | [atom()|binary()]. 39 | -type stats() :: [stat()]. 40 | -type stat() :: {stat_name(), stat_value()}. 41 | -type stat_name() :: list(). 42 | -type stat_value() :: integer() | [tuple()]. 43 | 44 | %% @doc To allow for namespacing, and adding richer dimensions, stats 45 | %% are named with a tuple key. The key (like `{riak_kv, node, gets}' or 46 | %% `{riak_kv, vnode, puts, time}') can 47 | %% be seen as an hierarchical path. With `riak_kv' at the root and 48 | %% the other elements as branches / leaves. 49 | %% This module allows us to get only the stats at and below a particular key. 50 | %% `Path' is a list of atoms or the empty list. 51 | %% an example path might be `[riak_kv]' which will return every 52 | %% stat that has `riak_kv' in the first element of its key tuple. 53 | %% You may use the atom '_' at any point 54 | %% in `Path' as a wild card. 55 | -spec get_stats(path()) -> stats(). 56 | get_stats(Path) -> 57 | exometer:get_values(Path). 58 | %% %% get all the stats that are at Path 59 | %% calculate_stats(exometer:select( 60 | %% [{ {Path ++ '_','_',enabled}, [], ['$_'] }])). 61 | 62 | calculate_stats(NamesAndTypes) -> 63 | [{Name, get_stat(Name)} || {Name, _, _} <- NamesAndTypes]. 64 | 65 | %% Create/lookup a cache/calculation process 66 | get_stat(Stat) -> 67 | exometer:get_value(Stat). 68 | 69 | %% Encapsulate getting a stat value from exometer. 70 | %% 71 | %% If for any reason we can't get a stats value 72 | %% return 'unavailable'. 73 | %% @TODO experience shows that once a stat is 74 | %% broken it stays that way. Should we delete 75 | %% stats that are broken? 76 | calc_stat({Name, _Type}) when is_tuple(Name) -> 77 | stat_return(exometer:get_value([riak_core_stat:prefix()|tuple_to_list(Name)])); 78 | calc_stat({[_|_] = Name, _Type}) -> 79 | stat_return(exometer:get_value([riak_core_stat:prefix()|Name])). 80 | 81 | stat_return({error,not_found}) -> unavailable; 82 | stat_return({ok, Value}) -> Value. 83 | 84 | log_error(StatName, ErrClass, ErrReason) -> 85 | ?LOG_WARNING("Failed to calculate stat ~p with ~p:~p", [StatName, ErrClass, ErrReason]). 86 | 87 | %% some crazy people put funs in gauges (exometer has a 'function' metric) 88 | %% so that they can have a consistent interface 89 | %% to access stats from disperate sources 90 | calc_gauge({function, Mod, Fun}) -> 91 | Mod:Fun(); 92 | calc_gauge(Val) -> 93 | Val. 94 | -------------------------------------------------------------------------------- /src/riak_core_stat_sup.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riak_core: Core Riak Application 4 | %% 5 | %% Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | -module(riak_core_stat_sup). 24 | 25 | -behaviour(supervisor). 26 | 27 | %% API 28 | -export([start_link/0]). 29 | 30 | %% Supervisor callbacks 31 | -export([init/1]). 32 | 33 | %% Helper macro for declaring children of supervisor 34 | -define(CHILD(I, Type, Timeout), {I, {I, start_link, []}, permanent, Timeout, Type, [I]}). 35 | -define(CHILD(I, Type), ?CHILD(I, Type, 5000)). 36 | 37 | %% =================================================================== 38 | %% API functions 39 | %% =================================================================== 40 | 41 | start_link() -> 42 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 43 | 44 | %% =================================================================== 45 | %% Supervisor callbacks 46 | %% =================================================================== 47 | 48 | init([]) -> 49 | %% Note that exometer_core starts folsom now, so it is not needed here 50 | Children = lists:flatten( 51 | [?CHILD(riak_core_stats_sup, supervisor)]), 52 | 53 | {ok, {{rest_for_one, 10, 10}, Children}}. 54 | -------------------------------------------------------------------------------- /src/riak_core_stat_xform.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | -module(riak_core_stat_xform). 21 | 22 | -export([parse_transform/2]). 23 | 24 | 25 | parse_transform(Forms, _) -> 26 | case os:getenv("RIAK_CORE_STAT_PREFIX") of 27 | false -> 28 | Forms; 29 | Str -> 30 | transform(Forms, list_to_atom(Str)) 31 | end. 32 | 33 | transform([{function,L,prefix,0,[_]}|T], Pfx) -> 34 | [{function, L, prefix, 0, 35 | [{clause, L, [], [], [{atom,L,Pfx}]}]}|T]; 36 | transform([H|T], Pfx) -> 37 | [H | transform(T, Pfx)]; 38 | transform([], _) -> 39 | []. 40 | 41 | 42 | -------------------------------------------------------------------------------- /src/riak_core_stats_sup.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% Copyright (c) 2007-2011 Basho Technologies, Inc. All Rights Reserved. 3 | %% 4 | %% This file is provided to you under the Apache License, 5 | %% Version 2.0 (the "License"); you may not use this file 6 | %% except in compliance with the License. You may obtain 7 | %% a copy of the License at 8 | %% 9 | %% http://www.apache.org/licenses/LICENSE-2.0 10 | %% 11 | %% Unless required by applicable law or agreed to in writing, 12 | %% software distributed under the License is distributed on an 13 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | %% KIND, either express or implied. See the License for the 15 | %% specific language governing permissions and limitations 16 | %% under the License. 17 | %% 18 | %% ------------------------------------------------------------------- 19 | -module(riak_core_stats_sup). 20 | -behaviour(supervisor). 21 | -export([start_link/0, init/1]). 22 | -export([start_server/1, stop_server/1]). 23 | 24 | %% Helper macro for declaring children of supervisor 25 | -define(CHILD(I, Type, Timeout), {I, {I, start_link, []}, permanent, Timeout, Type, [I]}). 26 | -define(CHILD(I, Type), ?CHILD(I, Type, 5000)). 27 | 28 | start_link() -> 29 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 30 | 31 | init([]) -> 32 | Children = [stat_server(Mod) || {_App, Mod} <- riak_core:stat_mods()], 33 | {ok, {{one_for_one, 5, 10}, [?CHILD(riak_core_stat_cache, worker)|Children]}}. 34 | 35 | start_server(Mod) -> 36 | Ref = stat_server(Mod), 37 | Pid = case supervisor:start_child(?MODULE, Ref) of 38 | {ok, Child} -> Child; 39 | {error, {already_started, Child}} -> Child 40 | end, 41 | Pid. 42 | 43 | stop_server(Mod) -> 44 | _ = supervisor:terminate_child(?MODULE, Mod), 45 | _ = supervisor:delete_child(?MODULE, Mod), 46 | ok. 47 | 48 | %% @private 49 | stat_server(Mod) -> 50 | ?CHILD(Mod, worker). 51 | -------------------------------------------------------------------------------- /src/riak_core_sup.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riak_core: Core Riak Application 4 | %% 5 | %% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | -module(riak_core_sup). 24 | 25 | -behaviour(supervisor). 26 | 27 | -include("riak_core_bg_manager.hrl"). 28 | 29 | %% API 30 | -export([start_link/0]). 31 | -export([ensembles_enabled/0]). 32 | 33 | %% Supervisor callbacks 34 | -export([init/1]). 35 | 36 | %% Helper macro for declaring children of supervisor 37 | -define(CHILD(I, Type, Timeout, Args), {I, {I, start_link, Args}, permanent, Timeout, Type, [I]}). 38 | -define(CHILD(I, Type, Timeout), ?CHILD(I, Type, Timeout, [])). 39 | -define(CHILD(I, Type), ?CHILD(I, Type, 5000)). 40 | 41 | %% =================================================================== 42 | %% API functions 43 | %% =================================================================== 44 | 45 | start_link() -> 46 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 47 | 48 | %% =================================================================== 49 | %% Supervisor callbacks 50 | %% =================================================================== 51 | 52 | init([]) -> 53 | DistMonEnabled = app_helper:get_env(riak_core, enable_dist_mon, 54 | true), 55 | {ok, Root} = application:get_env(riak_core, platform_data_dir), 56 | 57 | EnsembleSup = {riak_ensemble_sup, 58 | {riak_ensemble_sup, start_link, [Root]}, 59 | permanent, 30000, supervisor, [riak_ensemble_sup]}, 60 | 61 | Children = lists:flatten( 62 | [?CHILD(riak_core_bg_manager, worker), 63 | ?CHILD(riak_core_sysmon_minder, worker), 64 | ?CHILD(riak_core_vnode_sup, supervisor, 305000), 65 | ?CHILD(riak_core_eventhandler_sup, supervisor), 66 | [?CHILD(riak_core_dist_mon, worker) || DistMonEnabled], 67 | ?CHILD(riak_core_handoff_sup, supervisor), 68 | ?CHILD(riak_core_ring_events, worker), 69 | ?CHILD(riak_core_ring_manager, worker), 70 | ?CHILD(riak_core_metadata_manager, worker), 71 | ?CHILD(riak_core_metadata_hashtree, worker), 72 | ?CHILD(riak_core_broadcast, worker), 73 | ?CHILD(riak_core_vnode_proxy_sup, supervisor), 74 | ?CHILD(riak_core_node_watcher_events, worker), 75 | ?CHILD(riak_core_node_watcher, worker), 76 | ?CHILD(riak_core_vnode_manager, worker), 77 | ?CHILD(riak_core_capability, worker), 78 | ?CHILD(riak_core_gossip, worker), 79 | ?CHILD(riak_core_claimant, worker), 80 | ?CHILD(riak_core_table_owner, worker), 81 | ?CHILD(riak_core_stat_sup, supervisor), 82 | ?CHILD(riak_core_node_worker_pool_sup, supervisor), 83 | [EnsembleSup || ensembles_enabled()] 84 | ]), 85 | 86 | {ok, {{one_for_one, 10, 10}, Children}}. 87 | 88 | ensembles_enabled() -> 89 | Exists = (code:which(riak_ensemble_sup) =/= non_existing), 90 | Enabled = app_helper:get_env(riak_core, enable_consensus, false), 91 | Exists and Enabled. 92 | -------------------------------------------------------------------------------- /src/riak_core_sysmon_minder.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riak_core: Core Riak Application 4 | %% 5 | %% Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | -module(riak_core_sysmon_minder). 24 | 25 | -behaviour(gen_server). 26 | 27 | %% API 28 | -export([start_link/0]). 29 | 30 | %% gen_server callbacks 31 | -export([init/1, handle_call/3, handle_cast/2, handle_info/2, 32 | terminate/2, code_change/3]). 33 | 34 | -record(state, {}). 35 | 36 | %%%=================================================================== 37 | %%% API 38 | %%%=================================================================== 39 | 40 | %%-------------------------------------------------------------------- 41 | %% @doc 42 | %% Starts the server 43 | %% 44 | %% @spec start_link() -> {ok, Pid} | ignore | {error, Error} 45 | %% @end 46 | %%-------------------------------------------------------------------- 47 | start_link() -> 48 | gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). 49 | 50 | %%%=================================================================== 51 | %%% gen_server callbacks 52 | %%%=================================================================== 53 | 54 | %%-------------------------------------------------------------------- 55 | %% @private 56 | %% @doc 57 | %% Initializes the server 58 | %% 59 | %% @spec init(Args) -> {ok, State} | 60 | %% {ok, State, Timeout} | 61 | %% ignore | 62 | %% {stop, Reason} 63 | %% @end 64 | %%-------------------------------------------------------------------- 65 | init([]) -> 66 | %% Add our system_monitor event handler. We do that here because 67 | %% we have a process at our disposal (i.e. ourself) to receive the 68 | %% notification in the very unlikely event that the 69 | %% riak_core_sysmon_handler has crashed and been removed from the 70 | %% riak_sysmon_handler gen_event server. (If we had a supervisor 71 | %% or app-starting process add the handler, then if the handler 72 | %% crashes, nobody will act on the crash notification.) 73 | riak_core_sysmon_handler:add_handler(), 74 | 75 | {ok, #state{}}. 76 | 77 | %%-------------------------------------------------------------------- 78 | %% @private 79 | %% @doc 80 | %% Handling call messages 81 | %% 82 | %% @spec handle_call(Request, From, State) -> 83 | %% {reply, Reply, State} | 84 | %% {reply, Reply, State, Timeout} | 85 | %% {noreply, State} | 86 | %% {noreply, State, Timeout} | 87 | %% {stop, Reason, Reply, State} | 88 | %% {stop, Reason, State} 89 | %% @end 90 | %%-------------------------------------------------------------------- 91 | handle_call(_Request, _From, State) -> 92 | Reply = ok, 93 | {reply, Reply, State}. 94 | 95 | %%-------------------------------------------------------------------- 96 | %% @private 97 | %% @doc 98 | %% Handling cast messages 99 | %% 100 | %% @spec handle_cast(Msg, State) -> {noreply, State} | 101 | %% {noreply, State, Timeout} | 102 | %% {stop, Reason, State} 103 | %% @end 104 | %%-------------------------------------------------------------------- 105 | handle_cast(_Msg, State) -> 106 | {noreply, State}. 107 | 108 | %%-------------------------------------------------------------------- 109 | %% @private 110 | %% @doc 111 | %% Handling all non call/cast messages 112 | %% 113 | %% @spec handle_info(Info, State) -> {noreply, State} | 114 | %% {noreply, State, Timeout} | 115 | %% {stop, Reason, State} 116 | %% @end 117 | %%-------------------------------------------------------------------- 118 | handle_info({gen_event_EXIT, riak_core_sysmon_handler, _}, State) -> 119 | %% SASL will create an error message, no need for us to duplicate it. 120 | %% 121 | %% Our handler should never crash, but it did indeed crash. If 122 | %% there's a pathological condition somewhere that's generating 123 | %% lots of unforseen things that crash core's custom handler, we 124 | %% could make things worse by jumping back into the exploding 125 | %% volcano. Wait a little bit before jumping back. Besides, the 126 | %% system_monitor data is nice but is not critical: there is no 127 | %% need to make things worse if things are indeed bad, and if we 128 | %% miss a few seconds of system_monitor events, the world will not 129 | %% end. 130 | timer:sleep(2*1000), 131 | riak_core_sysmon_handler:add_handler(), 132 | {noreply, State}; 133 | handle_info(_Info, State) -> 134 | {noreply, State}. 135 | 136 | %%-------------------------------------------------------------------- 137 | %% @private 138 | %% @doc 139 | %% This function is called by a gen_server when it is about to 140 | %% terminate. It should be the opposite of Module:init/1 and do any 141 | %% necessary cleaning up. When it returns, the gen_server terminates 142 | %% with Reason. The return value is ignored. 143 | %% 144 | %% @spec terminate(Reason, State) -> void() 145 | %% @end 146 | %%-------------------------------------------------------------------- 147 | terminate(_Reason, _State) -> 148 | ok. 149 | 150 | %%-------------------------------------------------------------------- 151 | %% @private 152 | %% @doc 153 | %% Convert process state when code is changed 154 | %% 155 | %% @spec code_change(OldVsn, State, Extra) -> {ok, NewState} 156 | %% @end 157 | %%-------------------------------------------------------------------- 158 | code_change(_OldVsn, State, _Extra) -> 159 | {ok, State}. 160 | 161 | %%%=================================================================== 162 | %%% Internal functions 163 | %%%=================================================================== 164 | -------------------------------------------------------------------------------- /src/riak_core_table_owner.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% Copyright (c) 2016 Basho Technologies, Inc. All Rights Reserved. 3 | %% 4 | %% This file is provided to you under the Apache License, 5 | %% Version 2.0 (the "License"); you may not use this file 6 | %% except in compliance with the License. You may obtain 7 | %% a copy of the License at 8 | %% 9 | %% http://www.apache.org/licenses/LICENSE-2.0 10 | %% 11 | %% Unless required by applicable law or agreed to in writing, 12 | %% software distributed under the License is distributed on an 13 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | %% KIND, either express or implied. See the License for the 15 | %% specific language governing permissions and limitations 16 | %% under the License. 17 | %% 18 | %% ------------------------------------------------------------------- 19 | %% 20 | %% @doc

ETS tables, when created with the `ets:new/2' function, have a single 21 | %% owning process. If the owning process exits, then any ETS tables associated 22 | %% with that process are deleted. The purpose of the `riak_core_table_owner' 23 | %% module (which is a gen_server process) is to serve as the owning process for 24 | %% ETS tables that do not otherwise have an obvious owning process. For example, 25 | %% the `riak_core_throttle' module uses an ETS table for maintaining its state, 26 | %% but it is not itself a process and therefore the owning process for it ETS 27 | %% table is not clear. In this case, `riak_core_table_owner' can be used to 28 | %% create and own the ETS table on its behalf.

29 | %% 30 | %%

It is important that this process never crashes, as that would lead to 31 | %% loss of data. Therefore, a defensive approach is taken and any calls to 32 | %% external modules are protected with the try/catch mechanism.

33 | %% 34 | %%

Note that this first iteration does not provide any API functions for 35 | %% reading or writing data in ETS tables and therefore is appropriate only for 36 | %% named public ETS tables. In order to be more broadly useful, future 37 | %% enhancements to this module should include API functions for efficiently 38 | %% reading and writing ETS data, preferably without going through a gen_server 39 | %% call.

40 | -module(riak_core_table_owner). 41 | 42 | -behaviour(gen_server). 43 | 44 | %% API 45 | -export([start_link/0, 46 | create_table/2, 47 | maybe_create_table/2]). 48 | 49 | %% gen_server callbacks 50 | -export([init/1, 51 | handle_call/3, 52 | handle_cast/2, 53 | handle_info/2, 54 | terminate/2, 55 | code_change/3]). 56 | 57 | %% Unfortunately the `ets' module does not define a type for options, but we 58 | %% can at least insist on a list. 59 | -type ets_options() :: list(). 60 | -type ets_table() :: ets:tab(). 61 | -type create_table_result() :: {ok, ets_table()} | {error, Reason::term()}. 62 | 63 | %%%=================================================================== 64 | %%% API 65 | %%%=================================================================== 66 | 67 | -spec start_link() -> {ok, pid()} | ignore | {error, Reason::term()}. 68 | start_link() -> 69 | gen_server:start_link({local, ?MODULE}, ?MODULE, dict:new(), []). 70 | 71 | %% Creates a new ETS table with the given `Name' and `Options'. 72 | %% Since the table will be owned by the `riak_core_table_owner' process, it 73 | %% should be created with the `public' option so that other processes can read 74 | %% and write data in the table. 75 | -spec create_table(Name::atom(), Options::ets_options()) -> create_table_result(). 76 | create_table(Name, Options) -> 77 | gen_server:call(?MODULE, {create_table, Name, Options}). 78 | 79 | %% Creates a new ETS table with the given `Name' and `Options', if and only if 80 | %% it was not already created previously. 81 | %% Since the table will be owned by the `riak_core_table_owner' process, it 82 | %% should be created with the `public' option so that other processes can read 83 | %% and write data in the table. 84 | -spec maybe_create_table(Name::atom(), Options::ets_options()) -> create_table_result(). 85 | maybe_create_table(Name, Options) -> 86 | gen_server:call(?MODULE, {maybe_create_table, Name, Options}). 87 | 88 | %%%=================================================================== 89 | %%% gen_server callbacks 90 | %%%=================================================================== 91 | 92 | init(State) -> 93 | {ok, State}. 94 | 95 | handle_call({create_table, Name, Options}, _From, State) -> 96 | do_create_table(Name, Options, State); 97 | handle_call({maybe_create_table, Name, Options}, _From, State) -> 98 | case dict:find(Name, State) of 99 | {ok, Table} -> 100 | {reply, {ok, Table}, State}; 101 | error -> 102 | do_create_table(Name, Options, State) 103 | end. 104 | 105 | handle_cast(_Msg, State) -> 106 | {noreply, State}. 107 | 108 | handle_info(_Info, State) -> 109 | {noreply, State}. 110 | 111 | terminate(_Reason, _State) -> 112 | ok. 113 | 114 | code_change(_OldVsn, State, _Extra) -> 115 | {ok, State}. 116 | 117 | %%%=================================================================== 118 | %%% Internal functions 119 | %%%=================================================================== 120 | 121 | do_create_table(Name, Options, State) -> 122 | try 123 | Table = ets:new(Name, Options), 124 | {reply, {ok, Table}, dict:store(Name, Table, State)} 125 | catch 126 | Error -> 127 | {reply, {error, Error}, State} 128 | end. 129 | -------------------------------------------------------------------------------- /src/riak_core_test_util.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riak_test_util: utilities for test scripts 4 | %% 5 | %% Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | %% @doc utilities for test scripts 24 | 25 | -module(riak_core_test_util). 26 | 27 | -ifdef(TEST). 28 | -export([setup_mockring1/0, 29 | fake_ring/2, 30 | stop_pid/1, 31 | wait_for_pid/1]). 32 | -include_lib("eunit/include/eunit.hrl"). 33 | 34 | stop_pid(Other) when not is_pid(Other) -> 35 | ok; 36 | stop_pid(Pid) -> 37 | unlink(Pid), 38 | exit(Pid, shutdown), 39 | ok = wait_for_pid(Pid). 40 | 41 | wait_for_pid(Pid) -> 42 | Mref = erlang:monitor(process, Pid), 43 | receive 44 | {'DOWN', Mref, process, _, _} -> 45 | ok 46 | after 47 | 5000 -> 48 | {error, didnotexit} 49 | end. 50 | 51 | 52 | setup_mockring1() -> 53 | % requires a running riak_core_ring_manager, in test-mode is ok 54 | Ring0 = riak_core_ring:fresh(16,node()), 55 | Ring1 = riak_core_ring:add_member(node(), Ring0, 'othernode@otherhost'), 56 | Ring2 = riak_core_ring:add_member(node(), Ring1, 'othernode2@otherhost2'), 57 | 58 | Ring3 = lists:foldl(fun(_,R) -> 59 | riak_core_ring:transfer_node( 60 | hd(riak_core_ring:my_indices(R)), 61 | 'othernode@otherhost', R) end, 62 | Ring2,[1,2,3,4,5,6]), 63 | Ring = lists:foldl(fun(_,R) -> 64 | riak_core_ring:transfer_node( 65 | hd(riak_core_ring:my_indices(R)), 66 | 'othernode2@otherhost2', R) end, 67 | Ring3,[1,2,3,4,5,6]), 68 | riak_core_ring_manager:set_ring_global(Ring). 69 | 70 | fake_ring(Size, NumNodes) -> 71 | ManyNodes = [list_to_atom("dev" ++ integer_to_list(X) ++ "@127.0.0.1") 72 | || _ <- lists:seq(0, Size div NumNodes), 73 | X <- lists:seq(1, NumNodes)], 74 | Nodes = lists:sublist(ManyNodes, Size), 75 | Inc = chash:ring_increment(Size), 76 | Indices = lists:seq(0, (Size-1)*Inc, Inc), 77 | Owners = lists:zip(Indices, Nodes), 78 | [Node|OtherNodes] = Nodes, 79 | Ring = riak_core_ring:fresh(Size, Node), 80 | Ring2 = lists:foldl(fun(OtherNode, RingAcc) -> 81 | RingAcc2 = riak_core_ring:add_member(Node, RingAcc, OtherNode), 82 | riak_core_ring:set_member(Node, RingAcc2, OtherNode, 83 | valid, same_vclock) 84 | end, Ring, OtherNodes), 85 | Ring3 = lists:foldl(fun({Idx, Owner}, RingAcc) -> 86 | riak_core_ring:transfer_node(Idx, Owner, RingAcc) 87 | end, Ring2, Owners), 88 | Ring3. 89 | 90 | -endif. %TEST. 91 | -------------------------------------------------------------------------------- /src/riak_core_vnode_proxy_sup.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% Copyright (c) 2007-2011 Basho Technologies, Inc. All Rights Reserved. 3 | %% 4 | %% This file is provided to you under the Apache License, 5 | %% Version 2.0 (the "License"); you may not use this file 6 | %% except in compliance with the License. You may obtain 7 | %% a copy of the License at 8 | %% 9 | %% http://www.apache.org/licenses/LICENSE-2.0 10 | %% 11 | %% Unless required by applicable law or agreed to in writing, 12 | %% software distributed under the License is distributed on an 13 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | %% KIND, either express or implied. See the License for the 15 | %% specific language governing permissions and limitations 16 | %% under the License. 17 | %% 18 | %% ------------------------------------------------------------------- 19 | -module(riak_core_vnode_proxy_sup). 20 | -behaviour(supervisor). 21 | -export([start_link/0, init/1]). 22 | -export([start_proxy/2, stop_proxy/2, start_proxies/1]). 23 | 24 | -include_lib("kernel/include/logger.hrl"). 25 | 26 | start_link() -> 27 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 28 | 29 | init([]) -> 30 | %% Populate supervisor list with proxies for already registered vnode 31 | %% modules. Ensures restart of proxies after a crash of this supervisor. 32 | Indices = get_indices(), 33 | VMods = riak_core:vnode_modules(), 34 | Proxies = [proxy_ref(Mod, Index) || {_, Mod} <- VMods, 35 | Index <- Indices], 36 | {ok, {{one_for_one, 5, 10}, Proxies}}. 37 | 38 | start_proxy(Mod, Index) -> 39 | Ref = proxy_ref(Mod, Index), 40 | Pid = case supervisor:start_child(?MODULE, Ref) of 41 | {ok, Child} -> Child; 42 | {error, {already_started, Child}} -> Child 43 | end, 44 | Pid. 45 | 46 | stop_proxy(Mod, Index) -> 47 | _ = supervisor:terminate_child(?MODULE, {Mod, Index}), 48 | _ = supervisor:delete_child(?MODULE, {Mod, Index}), 49 | ok. 50 | 51 | start_proxies(Mod) -> 52 | ?LOG_DEBUG("Starting vnode proxies for: ~p", [Mod]), 53 | Indices = get_indices(), 54 | _ = [start_proxy(Mod, Index) || Index <- Indices], 55 | ok. 56 | 57 | %% @private 58 | proxy_ref(Mod, Index) -> 59 | {{Mod, Index}, {riak_core_vnode_proxy, start_link, [Mod, Index]}, 60 | permanent, 5000, worker, [riak_core_vnode_proxy]}. 61 | 62 | %% @private 63 | get_indices() -> 64 | {ok, Ring} = riak_core_ring_manager:get_raw_ring(), 65 | AllOwners = riak_core_ring:all_owners(Ring), 66 | [Idx || {Idx, _} <- AllOwners]. 67 | -------------------------------------------------------------------------------- /src/riak_core_vnode_sup.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riak_vnode_sup: supervise riak_vnode processes 4 | %% 5 | %% Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | %% @doc supervise riak_vnode processes 24 | 25 | -module(riak_core_vnode_sup). 26 | -behaviour(supervisor). 27 | -export([start_link/0, init/1]). 28 | -export([start_vnode/3]). 29 | 30 | start_vnode(Mod, Index, ForwardTo) when is_integer(Index) -> 31 | supervisor:start_child(?MODULE, [Mod, Index, ForwardTo]). 32 | 33 | start_link() -> 34 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 35 | 36 | %% @private 37 | init([]) -> 38 | {ok, 39 | {{simple_one_for_one, 10, 10}, 40 | [{undefined, 41 | {riak_core_vnode, start_link, []}, 42 | temporary, 300000, worker, dynamic}]}}. 43 | -------------------------------------------------------------------------------- /src/riak_core_vnode_worker.erl: -------------------------------------------------------------------------------- 1 | %% 2 | %% Copyright (c) 2007-2011 Basho Technologies, Inc. All Rights Reserved. 3 | %% 4 | %% This file is provided to you under the Apache License, 5 | %% Version 2.0 (the "License"); you may not use this file 6 | %% except in compliance with the License. You may obtain 7 | %% a copy of the License at 8 | %% 9 | %% http://www.apache.org/licenses/LICENSE-2.0 10 | %% 11 | %% Unless required by applicable law or agreed to in writing, 12 | %% software distributed under the License is distributed on an 13 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | %% KIND, either express or implied. See the License for the 15 | %% specific language governing permissions and limitations 16 | %% under the License. 17 | %% 18 | %% ------------------------------------------------------------------- 19 | -module(riak_core_vnode_worker). 20 | 21 | -behaviour(gen_server). 22 | 23 | -compile({nowarn_deprecated_function, 24 | [{gen_fsm, send_all_state_event, 2}]}). 25 | 26 | -include("riak_core_vnode.hrl"). 27 | 28 | -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, 29 | code_change/3]). 30 | -export([start_link/1, handle_work/3, handle_work/4]). 31 | 32 | -include_lib("kernel/include/logger.hrl"). 33 | 34 | -ifdef(PULSE). 35 | -compile(export_all). 36 | -compile({parse_transform, pulse_instrument}). 37 | -compile({pulse_replace_module, [{gen_fsm, pulse_gen_fsm}, 38 | {gen_server, pulse_gen_server}]}). 39 | -endif. 40 | 41 | -record(state, { 42 | module :: atom(), 43 | modstate :: any() 44 | }). 45 | 46 | -callback init_worker(VnodeIDx :: partition(), 47 | Args :: list(any()), 48 | Props :: list(tuple())) -> 49 | {ok, WorkerState :: any()}. 50 | 51 | -callback handle_work(Work :: any(), 52 | From :: sender(), 53 | WorkerState :: any()) -> 54 | {noreply, UpdWorkerState :: any()} | 55 | {reply, Reply :: any(), UpdWorkerState :: any()}. 56 | 57 | start_link(Args) -> 58 | WorkerMod = proplists:get_value(worker_callback_mod, Args), 59 | [VNodeIndex, WorkerArgs, WorkerProps, Caller] = proplists:get_value(worker_args, Args), 60 | gen_server:start_link(?MODULE, [WorkerMod, VNodeIndex, WorkerArgs, WorkerProps, Caller], []). 61 | 62 | handle_work(Worker, Work, From) -> 63 | handle_work(Worker, Work, From, self()). 64 | 65 | handle_work(Worker, Work, From, Caller) -> 66 | gen_server:cast(Worker, {work, Work, From, Caller}). 67 | 68 | init([Module, VNodeIndex, WorkerArgs, WorkerProps, Caller]) -> 69 | {ok, WorkerState} = Module:init_worker(VNodeIndex, WorkerArgs, WorkerProps), 70 | %% let the pool queue manager know there might be a worker to checkout 71 | gen_fsm:send_all_state_event(Caller, worker_start), 72 | {ok, #state{module=Module, modstate=WorkerState}}. 73 | 74 | handle_call(Event, _From, State) -> 75 | ?LOG_DEBUG("Vnode worker received synchronous event: ~p.", [Event]), 76 | {reply, ok, State}. 77 | 78 | handle_cast({work, Work, WorkFrom, Caller}, 79 | #state{module = Mod, modstate = ModState} = State) -> 80 | NewModState = case Mod:handle_work(Work, WorkFrom, ModState) of 81 | {reply, Reply, NS} -> 82 | riak_core_vnode:reply(WorkFrom, Reply), 83 | NS; 84 | {noreply, NS} -> 85 | NS 86 | end, 87 | %% check the worker back into the pool 88 | gen_fsm:send_all_state_event(Caller, {checkin, self()}), 89 | {noreply, State#state{modstate=NewModState}}; 90 | handle_cast(_Event, State) -> 91 | {noreply, State}. 92 | 93 | handle_info(_Info, State) -> 94 | {noreply, State}. 95 | 96 | terminate(_Reason, _State) -> 97 | ok. 98 | 99 | code_change(_OldVsn, State, _Extra) -> 100 | {ok, State}. 101 | 102 | -------------------------------------------------------------------------------- /src/riak_core_vnode_worker_pool.erl: -------------------------------------------------------------------------------- 1 | %% 2 | %% Copyright (c) 2007-2011 Basho Technologies, Inc. All Rights Reserved. 3 | %% 4 | %% This file is provided to you under the Apache License, 5 | %% Version 2.0 (the "License"); you may not use this file 6 | %% except in compliance with the License. You may obtain 7 | %% a copy of the License at 8 | %% 9 | %% http://www.apache.org/licenses/LICENSE-2.0 10 | %% 11 | %% Unless required by applicable law or agreed to in writing, 12 | %% software distributed under the License is distributed on an 13 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | %% KIND, either express or implied. See the License for the 15 | %% specific language governing permissions and limitations 16 | %% under the License. 17 | %% 18 | %% ------------------------------------------------------------------- 19 | 20 | %% @doc This is a wrapper around a poolboy pool, that implements a 21 | %% queue. That is, this process maintains a queue of work, and checks 22 | %% workers out of a poolboy pool to consume it. 23 | %% 24 | %% The workers it uses send two messages to this process. 25 | %% 26 | %% The first message is at startup, the bare atom 27 | %% `worker_started'. This is a clue to this process that a worker was 28 | %% just added to the poolboy pool, so a checkout request has a chance 29 | %% of succeeding. This is most useful after an old worker has exited - 30 | %% `worker_started' is a trigger guaranteed to arrive at a time that 31 | %% will mean an immediate poolboy:checkout will not beat the worker 32 | %% into the pool. 33 | %% 34 | %% The second message is when the worker finishes work it has been 35 | %% handed, the two-tuple, `{checkin, WorkerPid}'. This message gives 36 | %% this process the choice of whether to give the worker more work or 37 | %% check it back into poolboy's pool at this point. Note: the worker 38 | %% should *never* call poolboy:checkin itself, because that will 39 | %% confuse (or cause a race) with this module's checkout management. 40 | -module(riak_core_vnode_worker_pool). 41 | 42 | -behaviour(riak_core_worker_pool). 43 | 44 | -export([do_init/1, reply/2, do_work/3, to_log/0]). 45 | 46 | %% API 47 | -export([start_link/5, stop/2, shutdown_pool/2, handle_work/3]). 48 | 49 | start_link(WorkerMod, PoolSize, VNodeIndex, WorkerArgs, WorkerProps) -> 50 | riak_core_worker_pool:start_link( 51 | [WorkerMod, PoolSize, VNodeIndex, WorkerArgs, WorkerProps], 52 | ?MODULE, 53 | vnode_pool). 54 | 55 | handle_work(Pid, Work, From) -> 56 | riak_core_stat:update({worker_pool, vnode_pool}), 57 | riak_core_worker_pool:handle_work(Pid, Work, From). 58 | 59 | stop(Pid, Reason) -> 60 | riak_core_worker_pool:stop(Pid, Reason). 61 | 62 | %% wait for all the workers to finish any current work 63 | shutdown_pool(Pid, Wait) -> 64 | riak_core_worker_pool:shutdown_pool(Pid, Wait). 65 | 66 | reply(From, Msg) -> 67 | riak_core_vnode:reply(From, Msg). 68 | 69 | do_init([WorkerMod, PoolSize, VNodeIndex, WorkerArgs, WorkerProps]) -> 70 | poolboy:start_link([{worker_module, riak_core_vnode_worker}, 71 | {worker_args, 72 | [VNodeIndex, WorkerArgs, WorkerProps, self()]}, 73 | {worker_callback_mod, WorkerMod}, 74 | {size, PoolSize}, {max_overflow, 0}]). 75 | 76 | do_work(Pid, Work, From) -> 77 | riak_core_vnode_worker:handle_work(Pid, Work, From). 78 | 79 | to_log() -> false. -------------------------------------------------------------------------------- /test/13node_12node_ring.eqc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/basho/riak_core/25d9a6fa917eb8a2e95795d64eb88d7ad384ed88/test/13node_12node_ring.eqc -------------------------------------------------------------------------------- /test/169_group_join.eqc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/basho/riak_core/25d9a6fa917eb8a2e95795d64eb88d7ad384ed88/test/169_group_join.eqc -------------------------------------------------------------------------------- /test/648_unbalanced_singly.eqc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/basho/riak_core/25d9a6fa917eb8a2e95795d64eb88d7ad384ed88/test/648_unbalanced_singly.eqc -------------------------------------------------------------------------------- /test/claim-statem-leaving-nodes-still-claim.eqc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/basho/riak_core/25d9a6fa917eb8a2e95795d64eb88d7ad384ed88/test/claim-statem-leaving-nodes-still-claim.eqc -------------------------------------------------------------------------------- /test/claim_32_5_unbalanced.eqc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/basho/riak_core/25d9a6fa917eb8a2e95795d64eb88d7ad384ed88/test/claim_32_5_unbalanced.eqc -------------------------------------------------------------------------------- /test/claim_simulation.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2011 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | %% Based on an earlier edition provided by Greg Nelson and Ryan Zezeski: 22 | %% https://gist.github.com/992317 23 | 24 | -module(claim_simulation). 25 | -compile([export_all, nowarn_export_all]). 26 | 27 | %%-define(SIMULATE,1). 28 | -ifdef(SIMULATE). 29 | -ifdef(TEST). 30 | -include_lib("eunit/include/eunit.hrl"). 31 | 32 | -define(node(N), list_to_atom(lists:flatten(io_lib:format("riak@node~w",[N])))). 33 | -define(get(K, PL, D), proplists:get_value(K, PL, D)). 34 | 35 | basic_test_() -> 36 | {timeout, 60000, [fun basic_default/0]}. 37 | 38 | basic_default() -> 39 | Opts = [{suffix, "_default"}, 40 | {wc_mf, {riak_core_claim, default_wants_claim}}, 41 | {cc_mf, {riak_core_claim, default_choose_claim}}, 42 | {target_n_val, 4}, 43 | {ring_size, 32}, 44 | {node_count, 8}, 45 | {node_capacity, 24} 46 | ], 47 | run(Opts). 48 | 49 | run(Opts) -> 50 | application:load(riak_core), 51 | 52 | WCMod = ?get(wc_mf, Opts, {riak_core_claim, default_wants_claim}), 53 | CCMod = ?get(cc_mf, Opts, {riak_core_claim, default_choose_claim}), 54 | TargetN = ?get(target_n_val, Opts, 4), 55 | Suffix = ?get(suffix, Opts, ""), 56 | 57 | application:set_env(riak_core, wants_claim_fun, WCMod), 58 | application:set_env(riak_core, choose_claim_fun, CCMod), 59 | application:set_env(riak_core, target_n_val, TargetN), 60 | 61 | RingSize = ?get(ring_size, Opts, 2048), 62 | NodeCount = ?get(node_count, Opts, 100), 63 | NodeCapacity = ?get(node_capacity, Opts, 24), %% in TB 64 | 65 | Ring1 = riak_core_ring:fresh(RingSize, 'riak@node1'), 66 | {Rings, _} = lists:mapfoldl( 67 | fun(N, Prev) -> 68 | Node = ?node(N), 69 | R = riak_core_ring:add_member(Node, Prev, Node), 70 | Next = 71 | riak_core_gossip:claim_until_balanced(R, 72 | Node), 73 | {Next, Next} 74 | end, Ring1, lists:seq(2, NodeCount)), 75 | 76 | Owners1 = riak_core_ring:all_owners(Ring1), 77 | Owners = lists:map(fun riak_core_ring:all_owners/1, Rings), 78 | 79 | {Movers, _} = 80 | lists:mapfoldl( 81 | fun(Curr, Prev) -> 82 | Sum = length(lists:filter(fun not_same_node/1, 83 | lists:zip(Prev, Curr))), 84 | {Sum, Curr} 85 | end, Owners1, Owners), 86 | 87 | MeetTargetN = 88 | [riak_core_membership_claim:meets_target_n(R, TargetN) || R <- Rings], 89 | 90 | FName = io_lib:format("/tmp/rings_~w_~w~s.txt", 91 | [RingSize, NodeCount, Suffix]), 92 | {ok, Out} = file:open(FName, [write]), 93 | [print_info(Out, O, N, M, lists:nth(N - 1, MeetTargetN), 94 | RingSize, NodeCapacity) 95 | || {O, M, N} <- lists:zip3(Owners, Movers, lists:seq(2, NodeCount))], 96 | lists:foreach(fun(RingOut) -> 97 | riak_core_ring:pretty_print(RingOut, 98 | [{out, Out}, 99 | {target_n, TargetN}]) 100 | end, [Ring1|Rings]). 101 | 102 | not_same_node({{P,N}, {P,N}}) -> false; 103 | not_same_node({{P,_N}, {P,_M}}) -> true. 104 | 105 | print_info(Out, Owners, NodeCount, PartitionsMoved, MeetsTargetN, 106 | RingSize, NodeCapacity) -> 107 | Expect = round(RingSize / NodeCount), 108 | ActualPercent = 100 * (PartitionsMoved / RingSize), 109 | Terabytes = 0.8 * NodeCount * NodeCapacity * (PartitionsMoved / RingSize), 110 | %% Terabytes = Total Amount of Data * Fraction Moved 111 | 112 | F = fun({_,Node}, Acc) -> 113 | dict:update_counter(Node, 1, Acc) 114 | end, 115 | C = lists:keysort(1, dict:to_list(lists:foldl(F, dict:new(), Owners))), 116 | io:format(Out, 117 | "add node ~p, expect=~p, actual=~p, " 118 | "percent=~p, terabytes=~p, meets_target_n=~p~n" 119 | "counts: ~p~n~n", 120 | [NodeCount, Expect, PartitionsMoved, ActualPercent, 121 | Terabytes, MeetsTargetN,C]). 122 | 123 | -endif. 124 | -endif. 125 | -------------------------------------------------------------------------------- /test/my_ring: -------------------------------------------------------------------------------- 1 | {chstate,'dev5@127.0.0.1', 2 | [{'dev3@127.0.0.1',{89,63479179549}}, 3 | {'dev2@127.0.0.1',{79,63479091987}}, 4 | {'dev4@127.0.0.1',{43,63479013109}}, 5 | {'dev5@127.0.0.1',{43,63479013108}}, 6 | {'dev6@127.0.0.1',{130,63479013109}}], 7 | {64, 8 | [{0,'dev1@127.0.0.1'}, 9 | {22835963083295358096932575511191922182123945984,'dev2@127.0.0.1'}, 10 | {45671926166590716193865151022383844364247891968,'dev3@127.0.0.1'}, 11 | {68507889249886074290797726533575766546371837952,'dev4@127.0.0.1'}, 12 | {91343852333181432387730302044767688728495783936,'dev5@127.0.0.1'}, 13 | {114179815416476790484662877555959610910619729920,'dev6@127.0.0.1'}, 14 | {137015778499772148581595453067151533092743675904,'dev1@127.0.0.1'}, 15 | {159851741583067506678528028578343455274867621888,'dev2@127.0.0.1'}, 16 | {182687704666362864775460604089535377456991567872,'dev3@127.0.0.1'}, 17 | {205523667749658222872393179600727299639115513856,'dev4@127.0.0.1'}, 18 | {228359630832953580969325755111919221821239459840,'dev5@127.0.0.1'}, 19 | {251195593916248939066258330623111144003363405824,'dev6@127.0.0.1'}, 20 | {274031556999544297163190906134303066185487351808,'dev1@127.0.0.1'}, 21 | {296867520082839655260123481645494988367611297792,'dev2@127.0.0.1'}, 22 | {319703483166135013357056057156686910549735243776,'dev3@127.0.0.1'}, 23 | {342539446249430371453988632667878832731859189760,'dev4@127.0.0.1'}, 24 | {365375409332725729550921208179070754913983135744,'dev5@127.0.0.1'}, 25 | {388211372416021087647853783690262677096107081728,'dev6@127.0.0.1'}, 26 | {411047335499316445744786359201454599278231027712,'dev1@127.0.0.1'}, 27 | {433883298582611803841718934712646521460354973696,'dev2@127.0.0.1'}, 28 | {456719261665907161938651510223838443642478919680,'dev3@127.0.0.1'}, 29 | {479555224749202520035584085735030365824602865664,'dev4@127.0.0.1'}, 30 | {502391187832497878132516661246222288006726811648,'dev5@127.0.0.1'}, 31 | {525227150915793236229449236757414210188850757632,'dev6@127.0.0.1'}, 32 | {548063113999088594326381812268606132370974703616,'dev1@127.0.0.1'}, 33 | {570899077082383952423314387779798054553098649600,'dev2@127.0.0.1'}, 34 | {593735040165679310520246963290989976735222595584,'dev3@127.0.0.1'}, 35 | {616571003248974668617179538802181898917346541568,'dev4@127.0.0.1'}, 36 | {639406966332270026714112114313373821099470487552,'dev5@127.0.0.1'}, 37 | {662242929415565384811044689824565743281594433536,'dev6@127.0.0.1'}, 38 | {685078892498860742907977265335757665463718379520,'dev1@127.0.0.1'}, 39 | {707914855582156101004909840846949587645842325504,'dev2@127.0.0.1'}, 40 | {730750818665451459101842416358141509827966271488,'dev3@127.0.0.1'}, 41 | {753586781748746817198774991869333432010090217472,'dev4@127.0.0.1'}, 42 | {776422744832042175295707567380525354192214163456,'dev5@127.0.0.1'}, 43 | {799258707915337533392640142891717276374338109440,'dev6@127.0.0.1'}, 44 | {822094670998632891489572718402909198556462055424,'dev1@127.0.0.1'}, 45 | {844930634081928249586505293914101120738586001408,'dev2@127.0.0.1'}, 46 | {867766597165223607683437869425293042920709947392,'dev3@127.0.0.1'}, 47 | {890602560248518965780370444936484965102833893376,'dev4@127.0.0.1'}, 48 | {913438523331814323877303020447676887284957839360,'dev5@127.0.0.1'}, 49 | {936274486415109681974235595958868809467081785344,'dev6@127.0.0.1'}, 50 | {959110449498405040071168171470060731649205731328,'dev1@127.0.0.1'}, 51 | {981946412581700398168100746981252653831329677312,'dev2@127.0.0.1'}, 52 | {1004782375664995756265033322492444576013453623296, 53 | 'dev3@127.0.0.1'}, 54 | {1027618338748291114361965898003636498195577569280, 55 | 'dev4@127.0.0.1'}, 56 | {1050454301831586472458898473514828420377701515264, 57 | 'dev5@127.0.0.1'}, 58 | {1073290264914881830555831049026020342559825461248, 59 | 'dev6@127.0.0.1'}, 60 | {1096126227998177188652763624537212264741949407232, 61 | 'dev1@127.0.0.1'}, 62 | {1118962191081472546749696200048404186924073353216, 63 | 'dev2@127.0.0.1'}, 64 | {1141798154164767904846628775559596109106197299200, 65 | 'dev3@127.0.0.1'}, 66 | {1164634117248063262943561351070788031288321245184, 67 | 'dev4@127.0.0.1'}, 68 | {1187470080331358621040493926581979953470445191168, 69 | 'dev5@127.0.0.1'}, 70 | {1210306043414653979137426502093171875652569137152, 71 | 'dev6@127.0.0.1'}, 72 | {1233142006497949337234359077604363797834693083136, 73 | 'dev1@127.0.0.1'}, 74 | {1255977969581244695331291653115555720016817029120, 75 | 'dev2@127.0.0.1'}, 76 | {1278813932664540053428224228626747642198940975104, 77 | 'dev3@127.0.0.1'}, 78 | {1301649895747835411525156804137939564381064921088, 79 | 'dev4@127.0.0.1'}, 80 | {1324485858831130769622089379649131486563188867072, 81 | 'dev5@127.0.0.1'}, 82 | {1347321821914426127719021955160323408745312813056, 83 | 'dev6@127.0.0.1'}, 84 | {1370157784997721485815954530671515330927436759040, 85 | 'dev1@127.0.0.1'}, 86 | {1392993748081016843912887106182707253109560705024, 87 | 'dev2@127.0.0.1'}, 88 | {1415829711164312202009819681693899175291684651008, 89 | 'dev3@127.0.0.1'}, 90 | {1438665674247607560106752257205091097473808596992, 91 | 'dev4@127.0.0.1'}]}, 92 | {dict,0,16,16,8,80,48, 93 | {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}, 94 | {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}}}. 95 | -------------------------------------------------------------------------------- /test/riak_core_throttle_tests.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% Copyright (c) 2016 Basho Technologies, Inc. All Rights Reserved. 3 | %% 4 | %% This file is provided to you under the Apache License, 5 | %% Version 2.0 (the "License"); you may not use this file 6 | %% except in compliance with the License. You may obtain 7 | %% a copy of the License at 8 | %% 9 | %% http://www.apache.org/licenses/LICENSE-2.0 10 | %% 11 | %% Unless required by applicable law or agreed to in writing, 12 | %% software distributed under the License is distributed on an 13 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | %% KIND, either express or implied. See the License for the 15 | %% specific language governing permissions and limitations 16 | %% under the License. 17 | %% ------------------------------------------------------------------- 18 | -module(riak_core_throttle_tests). 19 | 20 | -include_lib("eunit/include/eunit.hrl"). 21 | 22 | -define(APP_NAME, test_app). 23 | 24 | activity_keys() -> 25 | [some_activity, another_activity, yet_another_activity]. 26 | 27 | clear_throttles(ActivityKeys) -> 28 | lists:foreach(fun(Key) -> 29 | riak_core_throttle:clear_throttle(?APP_NAME, Key), 30 | riak_core_throttle:clear_limits(?APP_NAME, Key), 31 | riak_core_throttle:enable_throttle(?APP_NAME, Key) 32 | end, 33 | ActivityKeys). 34 | 35 | throttle_test_() -> 36 | riak_core_table_owner:start_link(), 37 | riak_core_throttle:init(), 38 | {foreach, 39 | fun activity_keys/0, 40 | fun clear_throttles/1, 41 | [fun test_throttle_badkey/1, 42 | fun test_set_throttle/1, 43 | fun test_throttle_disable/1, 44 | fun test_set_throttle_by_load_with_no_limits/1, 45 | fun test_set_throttle_by_load_with_good_limits/1, 46 | fun test_set_throttle_by_load_actually_sets_throttle/1, 47 | fun test_set_limits_does_not_overwrite_current_throttle/1, 48 | fun test_set_limits_with_invalid_limits/1 49 | ]}. 50 | 51 | test_throttle_badkey([Key|_]) -> 52 | [?_assertError({badkey, Key}, riak_core_throttle:throttle(?APP_NAME, Key)), 53 | ?_assertEqual(undefined, riak_core_throttle:get_throttle(?APP_NAME, Key))]. 54 | 55 | test_set_throttle([Key1, Key2, Key3|_]) -> 56 | ok = riak_core_throttle:set_throttle(?APP_NAME, Key1, 42), 57 | ok = riak_core_throttle:set_throttle(?APP_NAME, Key2, 0), 58 | [?_assertEqual(42, riak_core_throttle:throttle(?APP_NAME, Key1)), 59 | ?_assertEqual(42, riak_core_throttle:get_throttle(?APP_NAME, Key1)), 60 | ?_assertEqual(0, riak_core_throttle:throttle(?APP_NAME, Key2)), 61 | ?_assertError({badkey, Key3}, riak_core_throttle:throttle(?APP_NAME, Key3))]. 62 | 63 | test_throttle_disable([Key1, Key2, Key3|_]) -> 64 | ok = riak_core_throttle:set_throttle(?APP_NAME, Key1, 42), 65 | ok = riak_core_throttle:set_throttle(?APP_NAME, Key2, 64), 66 | ok = riak_core_throttle:set_throttle(?APP_NAME, Key3, 11), 67 | ok = riak_core_throttle:disable_throttle(?APP_NAME, Key1), 68 | ok = riak_core_throttle:disable_throttle(?APP_NAME, Key2), 69 | ok = riak_core_throttle:enable_throttle(?APP_NAME, Key2), 70 | [?_assertNot(riak_core_throttle:is_throttle_enabled(?APP_NAME, Key1)), 71 | ?_assert(riak_core_throttle:is_throttle_enabled(?APP_NAME, Key2)), 72 | ?_assert(riak_core_throttle:is_throttle_enabled(?APP_NAME, Key3)), 73 | ?_assertEqual(0, riak_core_throttle:throttle(?APP_NAME, Key1)), 74 | ?_assertEqual(64, riak_core_throttle:throttle(?APP_NAME, Key2)), 75 | ?_assertEqual(11, riak_core_throttle:throttle(?APP_NAME, Key3))]. 76 | 77 | test_set_throttle_by_load_with_no_limits([Key|_]) -> 78 | [?_assertEqual(undefined, riak_core_throttle:get_limits(?APP_NAME, Key)), 79 | ?_assertError({no_limits, ?APP_NAME, Key}, 80 | riak_core_throttle:set_throttle_by_load(?APP_NAME, Key, 50))]. 81 | 82 | test_set_throttle_by_load_with_good_limits([Key, Key2|_]) -> 83 | Limits = [{-1, 0}, {10, 5}, {100, 10}], 84 | ok = riak_core_throttle:set_limits(?APP_NAME, Key, Limits), 85 | [?_assertEqual(Limits, riak_core_throttle:get_limits(?APP_NAME, Key)), 86 | ?_assertEqual(0, riak_core_throttle:set_throttle_by_load(?APP_NAME, Key, -10000000)), 87 | ?_assertEqual(0, riak_core_throttle:set_throttle_by_load(?APP_NAME, Key, -10)), 88 | ?_assertEqual(0, riak_core_throttle:set_throttle_by_load(?APP_NAME, Key, -1)), 89 | ?_assertEqual(0, riak_core_throttle:set_throttle_by_load(?APP_NAME, Key, 0)), 90 | ?_assertEqual(0, riak_core_throttle:set_throttle_by_load(?APP_NAME, Key, 1)), 91 | ?_assertEqual(0, riak_core_throttle:set_throttle_by_load(?APP_NAME, Key, 2)), 92 | ?_assertEqual(0, riak_core_throttle:set_throttle_by_load(?APP_NAME, Key, 9)), 93 | ?_assertEqual(5, riak_core_throttle:set_throttle_by_load(?APP_NAME, Key, 10)), 94 | ?_assertEqual(5, riak_core_throttle:set_throttle_by_load(?APP_NAME, Key, 11)), 95 | ?_assertEqual(5, riak_core_throttle:set_throttle_by_load(?APP_NAME, Key, 99)), 96 | ?_assertEqual(10, riak_core_throttle:set_throttle_by_load(?APP_NAME, Key, 100)), 97 | ?_assertEqual(10, riak_core_throttle:set_throttle_by_load(?APP_NAME, Key, 101)), 98 | ?_assertEqual(10, riak_core_throttle:set_throttle_by_load(?APP_NAME, Key, 10000001)), 99 | ?_assertEqual(10, riak_core_throttle:set_throttle_by_load(?APP_NAME, Key, max)), 100 | ?_assertError(function_clause, 101 | riak_core_throttle:set_throttle_by_load(?APP_NAME, Key, {max})), 102 | ?_assertError({no_limits, ?APP_NAME, Key2}, 103 | riak_core_throttle:set_throttle_by_load(?APP_NAME, Key2, 100)) 104 | ]. 105 | 106 | test_set_throttle_by_load_actually_sets_throttle([Key|_]) -> 107 | ok = riak_core_throttle:set_limits(?APP_NAME, Key, [{-1, 0}, {5, 42}]), 108 | 42 = riak_core_throttle:set_throttle_by_load(?APP_NAME, Key, 5), 109 | [?_assertEqual(42, riak_core_throttle:throttle(?APP_NAME, Key))]. 110 | 111 | test_set_limits_does_not_overwrite_current_throttle([Key|_]) -> 112 | ok = riak_core_throttle:set_limits(?APP_NAME, Key, [{-1, 0}, {100, 42}]), 113 | 42 = riak_core_throttle:set_throttle_by_load(?APP_NAME, Key, 500), 114 | ok = riak_core_throttle:set_limits(?APP_NAME, Key, [{-1, 0}, {100, 500}]), 115 | [?_assertEqual(42, riak_core_throttle:throttle(?APP_NAME, Key))]. 116 | 117 | test_set_limits_with_invalid_limits([Key|_]) -> 118 | [?_assertError(invalid_throttle_limits, 119 | riak_core_throttle:set_limits(?APP_NAME, Key, [{-1, 0}, {100, -1}])), 120 | ?_assertError(invalid_throttle_limits, 121 | riak_core_throttle:set_limits(?APP_NAME, Key, [{1, 0}, {100, 100}]))]. 122 | -------------------------------------------------------------------------------- /test/site1-cert.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIFFTCCAv2gAwIBAgIUXOs9PZNqebgYxs8u5wNrl0wnPGgwDQYJKoZIhvcNAQEL 3 | BQAwGjEYMBYGA1UEAwwPc2l0ZTEuYmFzaG8uY29tMB4XDTIzMDMxNDEwNDM1MloX 4 | DTMzMDMxMTEwNDM1MlowGjEYMBYGA1UEAwwPc2l0ZTEuYmFzaG8uY29tMIICIjAN 5 | BgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAku5lAk+VHXw128wmXnA9dPUtNTbQ 6 | V1/Q7OQNLmjGGCazZJZJL9o3yDIWzrKaPq6qN8LuKov4irx5orcXPaKvGh2RW626 7 | ssxWFtQtdn9qheNbKBpo2RGb1YxkSD8+0n3zz9XyTj3/tI8a9rZzTGz09P/TZ6pc 8 | jvPjaBve2T7V+2r1+G+lzVU/K9jrzBoV5tTP041Sl2QpJjuo78YWnuWoJedE6nez 9 | kXzTJtKtZvz0IujKuWdhhCSW6l3bZDB5qUDUuqjFRRBuCv8Me6mzOgubD8x7Kn+A 10 | m//yPEPOsVB3SbC46VNP3aRqqDPDMOuVendP5gCDGhbYpOzDd02l1nBt/VQtJEjq 11 | Gj2RtOoeqcYuNE51tjgdFrRsQqWOwvywkacvBtx+T8t0L4C3sfbybNFF1Q0WeKds 12 | n2+7XYHuvPqvgJoFduI42GZ3mYqJET9a9pNwAMjKtDxGUq3YCOuzNukeLykR1zAs 13 | RR2U6tejk7PFJ/7/MI/AzYfkEKhzDmkcCMk+89fSU4pwcSQ+kOr7I8+xTwblUV1n 14 | EJf9Ri0PT6Ep2i0vj90QQ68Ssjtw9aG4AUpYyc9SSK6hiosyXM49yR+5IKF9XCw8 15 | +xVLQFBC5by4FfynfgHQH7e+UvZCAjZFRVg/mYL++6T0HMsmCbmLK0wSAX2xIGcS 16 | V3Xhys6Kga5wRLkCAwEAAaNTMFEwHQYDVR0OBBYEFL2FKlZcj32OvROMOaY3SctH 17 | 9mBIMB8GA1UdIwQYMBaAFL2FKlZcj32OvROMOaY3SctH9mBIMA8GA1UdEwEB/wQF 18 | MAMBAf8wDQYJKoZIhvcNAQELBQADggIBAFutixNahjty9VHU10eI2jHC4w3H+L6c 19 | PVI6rft3007JWW8d+r4J+l/aGx8x+tzMTuj+ftD2rBAZ32knVC2FvPHcwSuwYMlD 20 | d7Of2KpWod+Bg3vqZCQg+5lwteJHE09cfN8zColP5oJB+7X459qKwDndhBR1FhL7 21 | kiWnSpLus7xM9XcFAOYvGkMjWxD/h77CNA7m9eQ97sZyFUUwuz56hjfGDRFeKFSy 22 | GHfDQNgG5zL5ctfRuB+P1IWIbND7244AtMqi/HGo66D3XnVXQG1FqrpokglUvwft 23 | YSb2XTHaJR8CewN/VjQUGeVx8mP3vTgsPs4k93lqCE+FiMxA+XCTULEk+HG0UXJz 24 | P1ZknLs2F8m96vUL5QUpV9kPwZAe7aujIuLBXTmJX1PnIdGfKwgTHJ6xXSrDOQQI 25 | ub3LrCl5xZmph9n2jQYOoioHIpza2HSyXOYeRWJjWbcLRSSpTWSN4TbqT1lL6ZHZ 26 | 44LMuV83kVzYaftpkoihgyaC6KvoDhwgLZQlQqf2Z5DE3aewihShmh1pczYyXIlo 27 | LiQlQ00c1ZZ6mFcmuffIslZE9CxWEjyHfjgN4l6TiF1ev8Mde9Yn72e0idkZ2HiN 28 | 1aSPj7EfgZtNxUrRay+8hJxm5Sfg1dA1ZfWLAYkM3b5oHiv1f8D6HlBp7ZNbmY5m 29 | VyWWe0yj710d 30 | -----END CERTIFICATE----- 31 | -------------------------------------------------------------------------------- /test/site1-key.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN PRIVATE KEY----- 2 | MIIJQQIBADANBgkqhkiG9w0BAQEFAASCCSswggknAgEAAoICAQCS7mUCT5UdfDXb 3 | zCZecD109S01NtBXX9Ds5A0uaMYYJrNklkkv2jfIMhbOspo+rqo3wu4qi/iKvHmi 4 | txc9oq8aHZFbrbqyzFYW1C12f2qF41soGmjZEZvVjGRIPz7SffPP1fJOPf+0jxr2 5 | tnNMbPT0/9NnqlyO8+NoG97ZPtX7avX4b6XNVT8r2OvMGhXm1M/TjVKXZCkmO6jv 6 | xhae5agl50Tqd7ORfNMm0q1m/PQi6Mq5Z2GEJJbqXdtkMHmpQNS6qMVFEG4K/wx7 7 | qbM6C5sPzHsqf4Cb//I8Q86xUHdJsLjpU0/dpGqoM8Mw65V6d0/mAIMaFtik7MN3 8 | TaXWcG39VC0kSOoaPZG06h6pxi40TnW2OB0WtGxCpY7C/LCRpy8G3H5Py3QvgLex 9 | 9vJs0UXVDRZ4p2yfb7tdge68+q+AmgV24jjYZneZiokRP1r2k3AAyMq0PEZSrdgI 10 | 67M26R4vKRHXMCxFHZTq16OTs8Un/v8wj8DNh+QQqHMOaRwIyT7z19JTinBxJD6Q 11 | 6vsjz7FPBuVRXWcQl/1GLQ9PoSnaLS+P3RBDrxKyO3D1obgBSljJz1JIrqGKizJc 12 | zj3JH7kgoX1cLDz7FUtAUELlvLgV/Kd+AdAft75S9kICNkVFWD+Zgv77pPQcyyYJ 13 | uYsrTBIBfbEgZxJXdeHKzoqBrnBEuQIDAQABAoICABZe+mgjVSaGc+4sTic7haVE 14 | yIZSpd63yv1WgknvpUGmGtLqLZMonCz8sLZLQUA2oy9sRuDu1ip2zzpJZYjP4piM 15 | /0XAf4Lw/lE+1aaLeIeRefPy2XCQxXOlNYY+gu0qRNtUeC5riCml+aADYC4MZNuC 16 | BrPA9OjFv3kUIpZcgoNjjLcvy0R710DGKANicV/nmmEGd8MZz/NvSqXUjjsmkmHQ 17 | WDuEJYYK8FyiinOkzfMcE0musOKdbjuZi2mHa+6rh/gyRsmLhTBiwYl5Pr2haLw7 18 | ZlauemwbcyFeiEjntyc0I2NQeQIxIhV2vssNriuq9jJPH6by6uIUM8qlZZDvJf/H 19 | X9kzDk3K5Khfsi5ajeERjtBn/+DkiYqLZSJh7cuBsQxhnHDxOZ4XkmLPRxCtj4NF 20 | 8Gfdm2xyUyIsMzZoHpM/rFpIhuLSD6164cbo5V1d09pnuv99XZ9Y8nIGATkgNj63 21 | hjORZEqMTRx9CL6H0BkGob9ioLm6APPl3WM7/gOQp5r4OfWKXrC4PMc7tLy2E+a7 22 | oxF6LiqSK/i1kKkUF6MqoiDsKPuIV5M4EqMwWKQDnrbsG5txT1j5NQiIgQKQnpf0 23 | pLpTew2h4krQBmAZXaDgswL8d1tryLhwsdSYpfZQVfgc5WeRcka+HfYZDo/0CXdF 24 | S7tJoWuhku+bM1JLLlzFAoIBAQDOjJ5vrUHhVmF+6LcNcPe93esB43JVAYUwAMLE 25 | Wl4lesUTLltaewDCZaHlYb3oWYntnov7wwYWxjoivXZAPtdN749nm3vCGIJLpiEI 26 | EYnDBxpN2yVF0TQE/f+6dgFrf3s7oo0CKlwq7lTy1qQ4b263rhI/XABUP75c2SaZ 27 | uAd9/aRtWZK1iZVnT99OEIikbtAl8rKMautwjxiiCeCSnVkYzAghljeQUN6lrl6e 28 | vmJWukdj0hMK4E4m1N2YVVqd8o6zH79emha5/hqFkaQTRGe8EHLA1EYe87NaGLiR 29 | nIk9gNpndeW0fAJmwtjhyOkbcrP6EDPdfCAJ7NgtzPnK/lnfAoIBAQC2G8qxi0uX 30 | cRw3S5VD3z1q1uhPXm9ZXBJP+j0q5a6FcbRbGbjKxUpRfpC/ogtu+iSnUrHXixFG 31 | 1qEk4nJeVBSBaqmvsRc1+tLMnlLTy8QL9ZS+X7xzBWN0CDvb774p+SP/O7YL1nwy 32 | GxE5mEaV1otpz/WOBjEYLEgQPEKUEY0BdOv6qp3YmUyZSuV+RG7H8Jc6qXBstwbc 33 | +yZc4gv7Njn0ldxaqd5ij+UXVUPEqXsPs6/i3NimGnnSrO7rF0KIwisN/6MKcFj9 34 | EgCOsA5ZTWESZ3yP85DtlF+N8O6WeW92rc1kgyoZksniJEK7mzOyPZtijvnKxs0s 35 | a0Z3g4CW4WRnAoIBADDwUHnVa06g0Ojo6rZcldZ+xjfkVCqf+ZauF3e+kZ//vsIU 36 | scB2dG+a9QkMZ5qt3GhYsIIuazrqLMweZ+5A9XKZ2DiMNn5+oCCcdCcdQK7c9/aK 37 | fgJU1yZ3qCfd3ehChEhU82QX9U6Wubnv8WkNvgX3UIuCmPFdSRVmviad2+iWOGdQ 38 | Wdp5htS7sAUnFbkgKGFVZ8itbKt2vDdob7VNYtblQOf0V6XyA07uSa0DHvm9mTyl 39 | ga2KB/pbZUW6s4XI+tamJwmbKNdYxZ/wKhgGlS2SGB5oLVUYGN64DMMze7D+3sC0 40 | xoBFAaynbgvRNqoyO0J8pE1OI6k2XOI9NtJKAbECggEAKjbXDcEUjM6Tk93ZhBpD 41 | ynh28a8acMR/GjsP4tnLUvPAsz2LMLX7PFjyhPmzXM5voLnyr9x07M7kHADCMkub 42 | VUS8gRW0I7c5FmW9HYumrn8HM/pNXPxqyYL/HUS9+Ezgo1HcKXJW5+aOX6OE+Ub6 43 | mlSyv3F///KD4f+WD2ziGuxXp4vVe1hOAVrzSAnlBEQpSlfsz8a9uIwfoHhl/p3b 44 | YW873WhzPLXOjVIem4AOHt2wcA5xyk9o3gVofOavbbRU9rHDwa+5cTCzIGB/cwQU 45 | 9Ic4wMXdWrWCUQ0AwbD0vbYcXqhpIfR9CAcP0R1xJmhRlM8/6FmdRia39wxsyo0F 46 | EQKCAQAbcqq4YAtrvwTDfz/7AKt9SHcMgB2x4ENj1axxfU23y0CX2nsxMnEEyUea 47 | /SAYVAOp4gkuFlY1PXjxbT4Ko4K4ZzERrvNgvvseT5l422xS0Kn5c7WkPBgUVLL8 48 | 9Tu/sICR/eCjDKD3vdn2AHeUO/OJHvf1TnS+y8sKQkpEu4yGc/P07NxOp5drLdRl 49 | f31Uz8oRGDeiwECI8XHTPFRmHuRGzHOoIJ+h3qrX4cVmPs0DEwVNQyV9GuG2xLsI 50 | Dp/qD3QfjNc5GOAZ2kvCBnMt5sGhTOeOFDjZCDL+6kFb0cSMES8K/dclpjV2fl/q 51 | 7kkoN2U3e6+/P7MS8sIo+dT5/2uI 52 | -----END PRIVATE KEY----- 53 | -------------------------------------------------------------------------------- /test/site2-cert.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIFFTCCAv2gAwIBAgIUWcZfOUg/E6MwI0bAsYLpB4tp+BEwDQYJKoZIhvcNAQEL 3 | BQAwGjEYMBYGA1UEAwwPc2l0ZTIuYmFzaG8uY29tMB4XDTIzMDMxNDEwNDQxM1oX 4 | DTMzMDMxMTEwNDQxM1owGjEYMBYGA1UEAwwPc2l0ZTIuYmFzaG8uY29tMIICIjAN 5 | BgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA+BwBP5wesN6nxSN3u8nGqq6mhBDW 6 | xGwKBQRRC1gYWGSkxDgv6khYKmIBUDYSVo0199B7PXrAnXoHyiJgiNps08EF5tWS 7 | CUQA/UXsMGvImKenPYI3Wfx89y3Ie+u/ibdCVj2ZDwIu4fOOODdkznerVptkHuJ9 8 | s7H/6Zs6KKZV54b2GJBEFXAJMTcdo2vRnUIRUXsn5pJUajkq1FVdTscvEDB29doF 9 | df0e3oPNWsE4c77Q/v4ZS/NUw/msHm5GWGUJRqzWqj+HIZnB0B+v4LKWqPAkS9Jz 10 | FjxUPP9ZBk8wbJmWVDgdxqW9LGJGctmNifnVxfHqOAb0ZK9uanr8LZ5vxhOxD0NN 11 | ivSMpnt8tuxN4Xm51QolSy9pgd6D8vqjnAt5u+zDdvJF14rvb35z3N1nYsyOQxJ6 12 | YncwEQ/sjELSmdO/OH3VAhLM0C+iWEYo0iRR1ZTMhTLFuIY79P2RFJh3RjsfjXjf 13 | X4A2gxvXfOLK3GW2umKG8OEe641HfNNGFhSsJnDjtc7yTbqwk38g6MaT/OEY3goK 14 | 1mO/UkMF8xTOrrS0rE71BZjp5tJwvrYhJwZUIYOrim8oYU8T3zKGeSVIZn9o7MWZ 15 | 16WOgW5D9HvIBgY/C2kvXn63Qa8G4cBfsCPWMCV6VgF/dmvvFPAuhEDzDmFfyPZY 16 | tUzO28hebK7sPv8CAwEAAaNTMFEwHQYDVR0OBBYEFN6yWPVSHY8j4/OhpCxdL1S7 17 | fcYkMB8GA1UdIwQYMBaAFN6yWPVSHY8j4/OhpCxdL1S7fcYkMA8GA1UdEwEB/wQF 18 | MAMBAf8wDQYJKoZIhvcNAQELBQADggIBAIAHYeji3dokWJh/QtrCJ7xe/8FKXZ/d 19 | gElkV2y6QbSYGxumUO0xlK1eNvL++lCoCOV13PK3XUHpIX5CuZuCFeigU0V3xXs8 20 | Cpb5WhsBW7W8h8SIXe/cPsU9X2dRcLydorXbgt8s+Z+TpVJ16K974u1d5Mxs8uqd 21 | ERiUB0cmMywXOLQ319bwwRm+aY7AkTBgVhQJ57os9p3iAIvMGabmJ65PH4LWM9ly 22 | IfwVahXpfY2DHCOyBKoi0FxCR/ltwfYi8SshswphTDP33p4h4mlHPlpIPr2UAet1 23 | 2/38gsgtBaQBkIiKYX5J9bnVs9frGNl4jTEum7VnMUZwpUZnrYGQ2iMITIHIvGJZ 24 | nZcw+xBSTkOuwb6vYhOztCag5GrtQ90SJHTsIMpPs1DoJFC5nNfvSI0qVZVwD+2K 25 | MhVkp4ik43qOLXCykcOvZi0VSQpm6iWwATOY6K05gVCdR05rVHdkVDL6nRR4t9Ms 26 | ZC/PkQ5TrdEptbOCzirr/VTNW+c4nsDFYTQU3YJFB4nGSp2Q+l0BVS4wYMWfDwpA 27 | T1nPgT3C/Bw5p1X+Zz8rEGviyWokbzMgzqgdee1JYlHDvO1jaVp2BnPxe4eRpfYC 28 | EcEPiiXzUlDxCSaZ//YtancJI4tHnydykcd0tWpFyiK7RSnsUzNobiRzJidmV211 29 | VGEetd9c7w5Q 30 | -----END CERTIFICATE----- 31 | -------------------------------------------------------------------------------- /test/site2-key.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN PRIVATE KEY----- 2 | MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQD4HAE/nB6w3qfF 3 | I3e7ycaqrqaEENbEbAoFBFELWBhYZKTEOC/qSFgqYgFQNhJWjTX30Hs9esCdegfK 4 | ImCI2mzTwQXm1ZIJRAD9Rewwa8iYp6c9gjdZ/Hz3Lch767+Jt0JWPZkPAi7h8444 5 | N2TOd6tWm2Qe4n2zsf/pmzooplXnhvYYkEQVcAkxNx2ja9GdQhFReyfmklRqOSrU 6 | VV1Oxy8QMHb12gV1/R7eg81awThzvtD+/hlL81TD+awebkZYZQlGrNaqP4chmcHQ 7 | H6/gspao8CRL0nMWPFQ8/1kGTzBsmZZUOB3Gpb0sYkZy2Y2J+dXF8eo4BvRkr25q 8 | evwtnm/GE7EPQ02K9Iyme3y27E3hebnVCiVLL2mB3oPy+qOcC3m77MN28kXXiu9v 9 | fnPc3WdizI5DEnpidzARD+yMQtKZ0784fdUCEszQL6JYRijSJFHVlMyFMsW4hjv0 10 | /ZEUmHdGOx+NeN9fgDaDG9d84srcZba6Yobw4R7rjUd800YWFKwmcOO1zvJNurCT 11 | fyDoxpP84RjeCgrWY79SQwXzFM6utLSsTvUFmOnm0nC+tiEnBlQhg6uKbyhhTxPf 12 | MoZ5JUhmf2jsxZnXpY6BbkP0e8gGBj8LaS9efrdBrwbhwF+wI9YwJXpWAX92a+8U 13 | 8C6EQPMOYV/I9li1TM7byF5sruw+/wIDAQABAoICAGt7+mB7umIGO2kwmdTdL3dO 14 | 7S1mfBwg7+BHg/KlqfpVM3ocG0nkKGnIBmig3CXpy/j9NP6n2J0AOjKr3OU8twuv 15 | YeakHD1uW/1dFtrNPtw/OTjw2hjeRKbyixmAQnyfSxpAWjgBypyxXinleYLmnfAJ 16 | poCovysjUEn8KOBmWu3P7noquYwN+rV6+piLA4ackhMaBumg9/gCpxbQmnkf2jgp 17 | vsd4OYhpbT+ZmtOXxGSERFdLblsUYeL6nUGbpVbzAmc/Rkn/9oMscKXArHlr9M/g 18 | JOFCHQCDhzzbpE4S7VgG23TF0MR3lbIOKxXjQKmG9cDosf6uqJ6bz3pKqEf9DM9I 19 | 455btNN/i130QcS0xo88neq3ML3V6qrWbjyysO8+rHZ4i2ddX6MmCzgy9PCTIepJ 20 | 1jh3yC52QDzkJyZRZAVEjRpNRiei+TA8yLmsoNdPEAeXwHTyPmXPSU4wxrJ7Kovj 21 | G9a7bW5wCN70Ae0LSkFkdW1eqfH7mtfEIuz9+JTaIROaqbwfEVTT+Y/LegcMPabK 22 | QIr24V7bIO9J+b2khdR/SO0CyjzpKZfULIF3F1uT/WpLhT2dg/TH8eC1W9tGA6WO 23 | +OsIdX5TaSM3DNHZUmfdHXQGUhALTlbgoU/+ei90FEc0/8Zs191vaSXlMkci7KN1 24 | 7t2KzhRE75VFEBnO/KgNAoIBAQD4snHej9QyoLGMIKrxemHw8ykjrX9Do/cm5AeV 25 | Ae7g8Oah4KDcvTKdC/tiy8Dpr0Ppx5fxV0dNRf0CChki6pePJUWrfppUlyIftNm6 26 | I+A7EK066DS7Mk2H3/5UEovsaht7hGGIM98bSev0AU7WM8zoxIdZp6jbaxwoWh2D 27 | i8b1DThlr3kNHtL5/ApbKoGRag7wWpwl75wo5R3w31oPsDfJS4LLpd6tqNDpDk9C 28 | SLeTM5dF/YOa/WH6kP+7kwHn69+cCFZDIRwtH29MOgVt5a61IaetgbZukXZGQAP1 29 | JRYUjg/xht+Hl3mEl0qXo30GGycM5Rfv3qmWpyvw3bwCtjijAoIBAQD/ZSR2RcVT 30 | RgHlP+7L7Efbys18n/UDd7z+dbltN81L5jSF8VBDdxoWUijP0dxyOGO9gCZkstOx 31 | W7FqhvKZWtO/7yB71dzf1uAlbI8oPUzozh2Xcckulh40ez9Zo114KgLdpwxVeeD+ 32 | 7+QFMNRji5Sa/LSYkweTCZsiFq55p7fTEs+7T/KmZJi3ny54zGu5ADn/clIShZCU 33 | QXKBhlGfa0BTkHkAwMCu3ULIAYDxt4JQb2J7ZRRlkB0JbCrR/i9/stkyYxcMrRXv 34 | yzrw9KQ+xn228o8Aj6+ZCFACp9DjY0SjBi74TPf0xVID3wxlLI1I+YYULqFygE/O 35 | YByvxFOAYXn1AoIBAE10bght7R2zdHh6+CJrBjPf13E4KMmtOF1RaZcb2gUOohXK 36 | eu8ksa29vJZhdPySzGjz87uDfj5vzMOk131LaNZ5OkCWFoS/6S3Myn6xi7V5lSoi 37 | hcHDgegOeAxBFnkT9uk6KpIb7aF3HViKS75kta3M/4aJtApaSpO6tTRyBQ3vByCM 38 | TPwsHHMk7yjDjUT3rh1navhZrt4BtZh+R3NZY5Z6wQqh5z9Ys2/SOBchxKLAz51J 39 | voI35hr+q9cEivRrPuI+qncPydG315b6/xPiE85v4+Qi4OmAvkBU0aa43S6brBWB 40 | 9v4oy/zijaW6kkajkD7w3sqjI1g7SnEi7do+XFcCggEBAMNF1SwtSqPSKk7bpDz9 41 | SzAtjxITx6XAHpw87OsqOGko+c3T5XfAEA0zAotrvW8D2Jci7bge1yeLoUp/JezF 42 | epQNRTwKGBh51eYnbAnrXOu/9PY/hQB+vVeJSLXD6AqhrJFHUXHlf3fNef3SCQKX 43 | zab3QypJeQge1yOoTf0slKvpfTWN+kzpK69yIXkxa8gG+VFsWn5Y62O/6spIJg6L 44 | VaLPGwXNW0YYXSBJrgqjMhtIHveUIJVLLDhxIQhwnma4G8Of6hbCHLy9GjL0CsUI 45 | xURlj5txQWTz6yEsqNXDp84yQyWMzCW13XCiUXrmJpy+VJsH0F+Y5ew4ppZpuSK2 46 | 5SUCggEAfqNMhRv8ckdVEMRD4E9koZjSORqidPy51VNodOJKHEWJlPIGHUS3e6aE 47 | ddcO7ngRXBvz8H95b5F+maQO4KckxxIlzgBENquRQqFwfd0941KuDMWOov+pbL4p 48 | dPK/yx62GLueXoHmbiV7HblRQLwRCX39eJ8ZJlVTWHJoumnGGPWl4bZkKsxl/I6N 49 | K6cfeJrNa9XnBgjm5r32cHRQEkVMsqK+/LOle44SSHe8G5exCV/eIrvuztOZO+ml 50 | T4HjTQGbV4lH7cy9VvxrJNgzmVGeVe3czdP8/+aLmkBb1AUON2IUAMCzxQDWjPiw 51 | Cg8Qq4BDEa41SASgce8o8D5MYH69vQ== 52 | -----END PRIVATE KEY----- 53 | -------------------------------------------------------------------------------- /test/sync_command_test.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2007-2011 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | -module(sync_command_test). 21 | 22 | -include_lib("eunit/include/eunit.hrl"). 23 | -include_lib("riak_core_vnode.hrl"). 24 | 25 | sync_test_() -> 26 | {foreach, 27 | fun setup_simple/0, 28 | fun stop_servers/1, 29 | [ {<<"Assert ok throw">>, 30 | fun() -> 31 | ?assertEqual(ok, mock_vnode:sync_error({0, node()}, goodthrow)) 32 | end}, 33 | {<<"Assert error throw">>, 34 | fun() -> 35 | ?assertEqual({error, terrible}, mock_vnode:sync_error({0, node()}, badthrow)) 36 | end}, 37 | {<<"Assert sync error">>, 38 | fun() -> 39 | ?assertError(core_breach, mock_vnode:sync_error({0, node()}, error)) 40 | end}, 41 | {<<"Assert sync exit">>, 42 | fun() -> 43 | ?assertError(core_breach, mock_vnode:sync_error({0, node()}, exit)) 44 | end}, 45 | {<<"Assert non-blocking sync error">>, 46 | fun() -> 47 | ?assertError(core_breach, mock_vnode:spawn_error({0, node()}, error)) 48 | end}, 49 | {<<"Assert non-blocking sync exit">>, 50 | fun() -> 51 | ?assertError(core_breach, mock_vnode:spawn_error({0, node()}, exit)) 52 | end} 53 | ] 54 | }. 55 | 56 | 57 | setup_simple() -> 58 | stop_servers(self()), 59 | Vars = [{ring_creation_size, 8}, 60 | {ring_state_dir, ""}, 61 | %% Don't allow rolling start of vnodes as it will cause a 62 | %% race condition with `all_nodes'. 63 | {core_vnode_eqc_pool_size, 0}, 64 | {vnode_rolling_start, 0}], 65 | error_logger:tty(false), 66 | _ = [begin 67 | Old = app_helper:get_env(riak_core, AppKey), 68 | ok = application:set_env(riak_core, AppKey, Val), 69 | {AppKey, Old} 70 | end || {AppKey, Val} <- Vars], 71 | exometer:start(), 72 | riak_core_ring_events:start_link(), 73 | riak_core_ring_manager:start_link(test), 74 | riak_core_vnode_proxy_sup:start_link(), 75 | 76 | {ok, _Sup} = riak_core_vnode_sup:start_link(), 77 | {ok, _} = riak_core_vnode_manager:start_link(), 78 | {ok, _VMaster} = riak_core_vnode_master:start_link(mock_vnode), 79 | 80 | 81 | ok = mock_vnode:start_vnode(0), 82 | %% NOTE: The return value from this call is currently not being 83 | %% used. This call is made to ensure that all msgs sent to 84 | %% the vnode mgr before this call have been handled. This 85 | %% guarantees that the vnode mgr ets tab is up-to-date 86 | 87 | riak_core:register([{vnode_module, mock_vnode}]). 88 | 89 | 90 | stop_servers(_Pid) -> 91 | %% Make sure VMaster is killed before sup as start_vnode is a cast 92 | %% and there may be a pending request to start the vnode. 93 | stop_pid(whereis(mock_vnode_master)), 94 | stop_pid(whereis(riak_core_vnode_manager)), 95 | stop_pid(whereis(riak_core_vnode_events)), 96 | stop_pid(whereis(riak_core_vnode_sup)), 97 | application:stop(exometer). 98 | 99 | stop_pid(undefined) -> 100 | ok; 101 | stop_pid(Pid) -> 102 | unlink(Pid), 103 | exit(Pid, shutdown), 104 | ok = wait_for_pid(Pid). 105 | 106 | wait_for_pid(Pid) -> 107 | Mref = erlang:monitor(process, Pid), 108 | receive 109 | {'DOWN',Mref,process,_,_} -> 110 | ok 111 | after 112 | 5000 -> 113 | {error, didnotexit} 114 | end. 115 | -------------------------------------------------------------------------------- /test/test_guarded_event_handler.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% test_guarded_event_handler 4 | %% 5 | %% Copyright (c) 2007-2011 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | -module(test_guarded_event_handler). 23 | -behaviour(gen_event). 24 | -export([start_link/0]). 25 | -export([init/1, handle_event/2, handle_call/2, 26 | handle_info/2, terminate/2, code_change/3]). 27 | -export([get_events/0]). 28 | -record(state, {events=[]}). 29 | 30 | -include_lib("eunit/include/eunit.hrl"). 31 | 32 | start_link() -> 33 | gen_event:start_link({local, ?MODULE}). 34 | 35 | get_events() -> 36 | gen_event:call(?MODULE, ?MODULE, get_events). 37 | 38 | init([]) -> 39 | {ok, #state{}}. 40 | 41 | handle_event({event, E}, State=#state{events=Events}) -> 42 | {ok, State#state{events=[E|Events]}}; 43 | handle_event(crash, State) -> 44 | exit(crash), 45 | {ok, State}. 46 | 47 | handle_call(get_events, State) -> 48 | {ok, State#state.events, State}. 49 | 50 | handle_info(_Info, State) -> 51 | {ok, State}. 52 | 53 | terminate(Reason, _State) -> 54 | Reason. 55 | 56 | code_change(_OldVsn, State, _Extra) -> 57 | {ok, State}. 58 | 59 | -ifdef(TEST). 60 | 61 | guarded_handler_test_() -> 62 | { setup, local, 63 | fun setup/0, 64 | fun cleanup/1, 65 | [ 66 | fun guarded_handler_test_case/0 67 | ] 68 | }. 69 | 70 | setup() -> 71 | riak_core_eventhandler_sup:start_link(), 72 | ?MODULE:start_link(). 73 | 74 | cleanup(_Pid) -> 75 | %% ARG: ugly hack to not die when the supervisor exits. 76 | process_flag(trap_exit, true), 77 | gen_event:stop(?MODULE). 78 | 79 | wait_for_exitfun() -> 80 | receive 81 | {?MODULE, {'EXIT', crash}} -> 82 | ok 83 | after 5000 -> 84 | fail 85 | end. 86 | 87 | guarded_handler_test_case() -> 88 | Self = self(), 89 | F = fun(Handler, Reason) -> 90 | Self ! {Handler, Reason} 91 | end, 92 | riak_core:add_guarded_event_handler(?MODULE, ?MODULE, [], F), 93 | gen_event:notify(?MODULE, {event, foo}), 94 | ?assertEqual(?MODULE:get_events(), [foo]), 95 | gen_event:notify(?MODULE, crash), 96 | ?assertEqual(wait_for_exitfun(), ok), 97 | wait_for_handler(?MODULE, 1000, 100), 98 | gen_event:notify(?MODULE, {event, baz}), 99 | ?assertEqual(?MODULE:get_events(), [baz]), 100 | ?assertEqual(riak_core:delete_guarded_event_handler(?MODULE,?MODULE,quux), quux), 101 | ?assertNot(lists:member(?MODULE, gen_event:which_handlers(?MODULE))), 102 | ?assertEqual([], supervisor:which_children(riak_core_eventhandler_sup)). 103 | 104 | wait_for_handler(_, 0, _) -> 105 | fail; 106 | wait_for_handler(Name, Count, Sleep) -> 107 | case lists:member(Name, gen_event:which_handlers(?MODULE)) of 108 | true -> ok; 109 | false -> 110 | timer:sleep(Sleep), 111 | wait_for_handler(Name, Count - 1, Sleep); 112 | _ -> 113 | ok 114 | end. 115 | 116 | -endif. 117 | -------------------------------------------------------------------------------- /test/worker_pool_test.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2007-2011 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | -module(worker_pool_test). 21 | 22 | -behaviour(riak_core_vnode_worker). 23 | -include_lib("eunit/include/eunit.hrl"). 24 | 25 | -export([init_worker/3, handle_work/3]). 26 | 27 | init_worker(_VnodeIndex, Noreply, _WorkerProps) -> 28 | {ok, Noreply}. 29 | 30 | handle_work(Work, From, true = State) -> 31 | Work(), 32 | riak_core_vnode:reply(From, ok), 33 | {noreply, State}; 34 | handle_work(Work, _From, false = State) -> 35 | Work(), 36 | {reply, ok, State}. 37 | 38 | -ifdef(TEST). 39 | 40 | receive_result(N) -> 41 | receive 42 | {N, ok} when N rem 2 /= 0 -> 43 | true; 44 | {N, {error, {worker_crash, _, _}}} when N rem 2 == 0 -> 45 | true 46 | after 47 | 0 -> 48 | timeout 49 | end. 50 | 51 | simple_worker_pool() -> 52 | {ok, Pool} = riak_core_vnode_worker_pool:start_link(?MODULE, 3, 10, false, []), 53 | [ riak_core_vnode_worker_pool:handle_work(Pool, fun() -> 54 | timer:sleep(100), 55 | 1/(N rem 2) 56 | end, 57 | {raw, N, self()}) 58 | || N <- lists:seq(1, 10)], 59 | 60 | timer:sleep(1200), 61 | 62 | %% make sure we got all the expected responses 63 | 64 | [ ?assertEqual(true, receive_result(N)) || N <- lists:seq(1, 10)], 65 | unlink(Pool), 66 | riak_core_vnode_worker_pool:stop(Pool, normal). 67 | 68 | simple_node_worker_pool() -> 69 | {ok, BestEffortPool} = 70 | riak_core_node_worker_pool:start_link(?MODULE, 71 | 1, 72 | false, 73 | [], 74 | be_pool), 75 | [ riak_core_node_worker_pool:handle_work(be_pool, fun() -> 76 | timer:sleep(100), 77 | 1/(N rem 2) 78 | end, 79 | {raw, N, self()}) 80 | || N <- lists:seq(1, 10)], 81 | 82 | timer:sleep(1000 + 500), 83 | 84 | %% make sure we got all the expected responses 85 | 86 | [ ?assertEqual(true, receive_result(N)) || N <- lists:seq(1, 10)], 87 | unlink(BestEffortPool), 88 | riak_core_node_worker_pool:stop(BestEffortPool, normal), 89 | 90 | {ok, AssuredForwardingPool} = 91 | riak_core_node_worker_pool:start_link(?MODULE, 92 | 5, 93 | false, 94 | [], 95 | af1_pool), 96 | [ riak_core_node_worker_pool:handle_work( 97 | af1_pool, 98 | fun() -> 99 | timer:sleep(100), 100 | 1/(N rem 2) 101 | end, 102 | {raw, N, self()}) 103 | || N <- lists:seq(1, 10)], 104 | 105 | timer:sleep(200 + 500), 106 | 107 | %% make sure we got all the expected responses 108 | 109 | [ ?assertEqual(true, receive_result(N)) || N <- lists:seq(1, 10)], 110 | unlink(AssuredForwardingPool), 111 | riak_core_node_worker_pool:stop(AssuredForwardingPool, normal). 112 | 113 | simple_noreply_worker_pool() -> 114 | {ok, Pool} = 115 | riak_core_vnode_worker_pool:start_link(?MODULE, 3, 10, true, []), 116 | [ riak_core_vnode_worker_pool:handle_work(Pool, fun() -> 117 | timer:sleep(100), 118 | 1/(N rem 2) 119 | end, 120 | {raw, N, self()}) 121 | || N <- lists:seq(1, 10)], 122 | 123 | timer:sleep(1200), 124 | 125 | %% make sure we got all the expected responses 126 | 127 | [ ?assertEqual(true, receive_result(N)) || N <- lists:seq(1, 10)], 128 | unlink(Pool), 129 | riak_core_vnode_worker_pool:stop(Pool, normal). 130 | 131 | 132 | pool_test_() -> 133 | {setup, 134 | fun() -> 135 | error_logger:tty(false) 136 | end, 137 | fun(_) -> 138 | error_logger:tty(true) 139 | end, 140 | [ 141 | fun simple_worker_pool/0, 142 | fun simple_noreply_worker_pool/0, 143 | fun simple_node_worker_pool/0 144 | ] 145 | }. 146 | 147 | -endif. 148 | --------------------------------------------------------------------------------