├── .gitignore ├── .hgignore ├── .hgtags ├── .thumbs.yml ├── .travis.yml ├── LICENSE ├── Makefile ├── README.md ├── contrib ├── Makefile └── handoff_perftool.erl ├── docs ├── aae.md ├── balanced.png ├── claim-fixes.md ├── hashtree.md ├── hashtree.png ├── partitioning.md ├── ring-resizing.md └── unbalanced.png ├── eqc ├── bg_manager_eqc.erl ├── btypes_eqc.erl ├── bucket_eqc_utils.erl ├── chash_eqc.erl ├── core_vnode_eqc.erl ├── hashtree_eqc.erl ├── mock_vnode.erl ├── new_cluster_membership_model_eqc.erl └── riak_core_ring_eqc.erl ├── include ├── riak_core.hrl ├── riak_core_bg_manager.hrl ├── riak_core_bucket_type.hrl ├── riak_core_dtrace.hrl ├── riak_core_handoff.hrl ├── riak_core_metadata.hrl ├── riak_core_ring.hrl └── riak_core_vnode.hrl ├── mkcerts.sh ├── priv └── riak_core.schema ├── rebar.config ├── rebar.lock ├── rebar3 ├── src ├── app_helper.erl ├── bloom.erl ├── dvvset.erl ├── gen_nb_server.erl ├── hashtree.erl ├── hashtree_tree.erl ├── process_proxy.erl ├── riak_core.app.src ├── riak_core.erl ├── riak_core_aae_vnode.erl ├── riak_core_apl.erl ├── riak_core_app.erl ├── riak_core_base64url.erl ├── riak_core_bg_manager.erl ├── riak_core_broadcast.erl ├── riak_core_broadcast_handler.erl ├── riak_core_bucket.erl ├── riak_core_bucket_props.erl ├── riak_core_bucket_type.erl ├── riak_core_capability.erl ├── riak_core_cinfo_core.erl ├── riak_core_claim.erl ├── riak_core_claim_sim.erl ├── riak_core_claim_util.erl ├── riak_core_claimant.erl ├── riak_core_cli_registry.erl ├── riak_core_cluster_cli.erl ├── riak_core_console.erl ├── riak_core_console_table.erl ├── riak_core_coverage_fsm.erl ├── riak_core_coverage_plan.erl ├── riak_core_dist_mon.erl ├── riak_core_dtrace.erl ├── riak_core_entropy_info.erl ├── riak_core_entropy_manager.erl ├── riak_core_eventhandler_guard.erl ├── riak_core_eventhandler_sup.erl ├── riak_core_exchange_fsm.erl ├── riak_core_exo_monitor.erl ├── riak_core_format.erl ├── riak_core_gen_server.erl ├── riak_core_gossip.erl ├── riak_core_handoff_cli.erl ├── riak_core_handoff_listener.erl ├── riak_core_handoff_listener_sup.erl ├── riak_core_handoff_manager.erl ├── riak_core_handoff_receiver.erl ├── riak_core_handoff_receiver_sup.erl ├── riak_core_handoff_sender.erl ├── riak_core_handoff_sender_sup.erl ├── riak_core_handoff_status.erl ├── riak_core_handoff_sup.erl ├── riak_core_index_hashtree.erl ├── riak_core_metadata.erl ├── riak_core_metadata_evt_sup.erl ├── riak_core_metadata_exchange_fsm.erl ├── riak_core_metadata_hashtree.erl ├── riak_core_metadata_manager.erl ├── riak_core_metadata_object.erl ├── riak_core_mochiglobal.erl ├── riak_core_net_ticktime.erl ├── riak_core_new_claim.erl ├── riak_core_node_watcher.erl ├── riak_core_node_watcher_events.erl ├── riak_core_nodeid.erl ├── riak_core_priority_queue.erl ├── riak_core_pw_auth.erl ├── riak_core_rand.erl ├── riak_core_repair.erl ├── riak_core_ring.erl ├── riak_core_ring_events.erl ├── riak_core_ring_handler.erl ├── riak_core_ring_manager.erl ├── riak_core_ring_util.erl ├── riak_core_security.erl ├── riak_core_send_msg.erl ├── riak_core_ssl_util.erl ├── riak_core_stat.erl ├── riak_core_stat_cache.erl ├── riak_core_stat_calc_proc.erl ├── riak_core_stat_calc_sup.erl ├── riak_core_stat_q.erl ├── riak_core_stat_sup.erl ├── riak_core_stat_xform.erl ├── riak_core_stats_sup.erl ├── riak_core_status.erl ├── riak_core_sup.erl ├── riak_core_sysmon_handler.erl ├── riak_core_sysmon_minder.erl ├── riak_core_table_owner.erl ├── riak_core_tcp_mon.erl ├── riak_core_test_util.erl ├── riak_core_throttle.erl ├── riak_core_tracer.erl ├── riak_core_util.erl ├── riak_core_vnode.erl ├── riak_core_vnode_manager.erl ├── riak_core_vnode_master.erl ├── riak_core_vnode_proxy.erl ├── riak_core_vnode_proxy_sup.erl ├── riak_core_vnode_sup.erl ├── riak_core_vnode_worker.erl ├── riak_core_vnode_worker_pool.erl ├── supervisor_pre_r14b04.erl └── vclock.erl ├── test ├── 13node_12node_ring.eqc ├── 169_group_join.eqc ├── 648_unbalanced_singly.eqc ├── bg_manager_tests.erl ├── bprops_eqc.erl ├── bucket_eqc_utils.erl ├── bucket_fixup_test.erl ├── claim-statem-leaving-nodes-still-claim.eqc ├── claim_32_5_unbalanced.eqc ├── claim_simulation.erl ├── dummy_evt.erl ├── mock_vnode.erl ├── my_ring ├── node_watcher_qc.erl ├── riak_core_base64url_tests.erl ├── riak_core_capability_tests.erl ├── riak_core_claim_statem.erl ├── riak_core_schema_tests.erl ├── riak_core_security_tests.erl ├── riak_core_throttle_tests.erl ├── site1-cert.pem ├── site1-key.pem ├── site2-cert.pem ├── site2-key.pem ├── sync_command_test.erl ├── test_guarded_event_handler.erl ├── vclock_qc.erl ├── worker_pool_pulse.erl └── worker_pool_test.erl └── tools.mk /.gitignore: -------------------------------------------------------------------------------- 1 | .eunit/* 2 | log 3 | deps/* 4 | priv/* 5 | ebin/* 6 | *.o 7 | include/*_pb.hrl 8 | *~ 9 | doc/* 10 | /.eqc-info 11 | /current_counterexample.eqc 12 | .local_dialyzer_plt 13 | dialyzer_warnings 14 | dialyzer_unhandled_warnings 15 | .rebar 16 | 17 | _build/ 18 | data/ 19 | nonode@nohost/ 20 | ring_manager_eunit/ 21 | sim.out 22 | .eqc 23 | compile_commands.json 24 | rebar3.crashdump 25 | .rebar3/ 26 | core_vnode_eqc.log 27 | -------------------------------------------------------------------------------- /.hgignore: -------------------------------------------------------------------------------- 1 | syntax regex 2 | .*~ 3 | ^ebin/.*.beam 4 | include/riak_core_pb.hrl 5 | ^.eunit -------------------------------------------------------------------------------- /.hgtags: -------------------------------------------------------------------------------- 1 | 1be0ce0200a7ac51cc3e0ca7ee9c244a539e409f riak_core-0.13.0rc1 2 | 1e43d42258bcc02b5797ee73f02fae60bfeac67e riak_core-0.13.0rc2 3 | 69b0f62c1f01b7a3c48d9379f6fe0e152331deee riak_core-0.13.0rc3 4 | 82ebfed1e301bff550f9690de874ca1ffb55cd17 riak_core-0.13.0rc5 5 | 1048210d6c679d3599b362b6df9a0002cf3c60a7 riak_core-0.13.0rc6 6 | c1b5e936c950aa4e29b4ffdc760e5a2651349018 riak_core-0.13.0rc7 7 | cb8c8703125a21e2eb3454236449ca957b26bacb riak_core-0.13.0rc8 8 | 063f7cdf287721f4d160c0c99192667a653a237f riak_core-0.13.0rc9 9 | 5b2683c697e6f322920fd276177c08764c722c98 riak_core-0.13.0 10 | -------------------------------------------------------------------------------- /.thumbs.yml: -------------------------------------------------------------------------------- 1 | minimum_reviewers: 2 2 | merge: true 3 | build_steps: 4 | - make clean 5 | - make deps 6 | - make compile 7 | - make test 8 | - make xref 9 | - make dialyzer 10 | org_mode: true 11 | timeout: 1800 12 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: erlang 2 | notifications: 3 | webhooks: http://basho-engbot.herokuapp.com/travis?key=72ce513a4a26166521f60f72511bfb905329db87 4 | email: eng@basho.com 5 | otp_release: 6 | - R16B 7 | - R15B03 8 | - R15B02 9 | - R15B01 10 | - R15B 11 | 12 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | DIALYZER_APPS = kernel stdlib sasl erts ssl tools os_mon runtime_tools crypto inets \ 2 | public_key mnesia syntax_tools compiler 3 | PULSE_TESTS = worker_pool_pulse 4 | 5 | .PHONY: deps test 6 | 7 | all: compile 8 | 9 | compile: deps 10 | ./rebar3 compile 11 | 12 | clean: 13 | ./rebar3 clean 14 | 15 | distclean: clean 16 | 17 | # You should 'clean' before your first run of this target 18 | # so that deps get built with PULSE where needed. 19 | pulse: 20 | ./rebar compile -D PULSE 21 | ./rebar eunit -D PULSE skip_deps=true suite=$(PULSE_TESTS) 22 | 23 | include tools.mk 24 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # What is riak_core_ng? 2 | 3 | The 'ng' version of riak core is a modernized version of riak core, it is entirely build with rebar3 using hex packages for all dependences, with more modern versions of some libraries. 99.9% of all credit goes to basho, very little extra work has happened here! The aim is to keep this up to date with all changes pushed to riak_core proper. 4 | 5 | ## Improvements and Additions 6 | 7 | * R19 compatibility - This form works with R18 and R19 (lower versions should work but are not tested) 8 | * AAE - this version includes the Active Anti Entropy code extracted from `riak_kv` and provides a new behavior called `riak_core_aae_vnode`. 9 | 10 | # Riak Core 11 | 12 | [![Build Status](https://secure.travis-ci.org/basho/riak_core.png)](http://travis-ci.org/basho/riak_core) 13 | 14 | Riak Core is the distributed systems framework that forms the basis of 15 | how [Riak](http://github.com/basho/riak) distributes data and scales. 16 | More generally, it can be thought of as a toolkit for building 17 | distributed, scalable, fault-tolerant applications. 18 | 19 | For some introductory reading on Riak Core (that’s not pure code), 20 | there’s an old but still valuable 21 | [blog post on the Basho Blog](http://basho.com/where-to-start-with-riak-core/) 22 | that’s well worth your time. 23 | 24 | ## Contributing 25 | 26 | We love community code, bug fixes, and other forms of contribution. We 27 | use GitHub Issues and Pull Requests for contributions to this and all 28 | other code. To get started: 29 | 30 | 1. Fork this repository. 31 | 2. Clone your fork or add the remote if you already have a clone of 32 | the repository. 33 | 3. Create a topic branch for your change. 34 | 4. Make your change and commit. Use a clear and descriptive commit 35 | message, spanning multiple lines if detailed explanation is needed. 36 | 5. Push to your fork of the repository and then send a pull request. 37 | 38 | 6. A Riak committer will review your patch and merge it into the main 39 | repository or send you feedback. 40 | 41 | ## Issues, Questions, and Bugs 42 | 43 | There are numerous ways to file issues or start conversations around 44 | something Core related 45 | 46 | * The 47 | [Riak Users List](http://lists.basho.com/mailman/listinfo/riak-users_lists.basho.com) 48 | is the main place for all discussion around Riak. 49 | * There is a 50 | [Riak Core-specific mailing list](http://lists.basho.com/mailman/listinfo/riak-core_lists.basho.com) 51 | for issues and questions that pertain to Core but not Riak. 52 | * #riak on Freenode is a very active channel and can get you some 53 | real-time help if a lot of instances. 54 | * If you've found a bug in Riak Core, 55 | [file](https://github.com/basho/riak_core/issues) a clear, concise, 56 | explanatory issue against this repo. 57 | -------------------------------------------------------------------------------- /contrib/Makefile: -------------------------------------------------------------------------------- 1 | 2 | 3 | INC=-I $(CURDIR)/../deps/riak_core/include/ -I $(CURDIR)/../include/ 4 | ERLC=erlc 5 | 6 | BUILD=$(ERLC) $(INC) 7 | 8 | all: 9 | @echo $(CURDIR) 10 | $(BUILD) handoff_perftool.erl 11 | -------------------------------------------------------------------------------- /docs/aae.md: -------------------------------------------------------------------------------- 1 | # AAE 2 | 3 | 4 | ## Quickstart: 5 | 6 | * Add the Entropy Manager to your apps supervisor tree. 7 | * Add the `riak_core_aae_vnode` behavior to the vnode in question. Implement the functions required by the behavior. 8 | * Implement the callback matches detailed in the VNode section. 9 | * Make sure that `init/1` creates a hashtree, see `maybe_create_hashtrees/2` as example how to do that. 10 | * Make all data changing calls in your VNode also call `update_hashtree/3` (or something similar) or `riak_core_index_hashtree:delete`. 11 | * Add the required config parameters. 12 | 13 | 14 | ## Entropy Manger (`riak_core_entropy_manager`) 15 | One is started per vnode service, as part of the of the main apps supervisor. It needs the service name and the vnode module passed as arguments. 16 | 17 | Here an example how to start it for the `snarl_user` service that is implemented in the `snarl_user_vnode`. 18 | 19 | ```erlang 20 | EntropyManagerUser = riak_core_entropy_manager:supervisor_spec(snarl_user, snarl_user_vnode), 21 | ``` 22 | 23 | The manager is responsible of coordinating the AAE tree build and exchange with other nodes. 24 | 25 | ## Entropy Info (`riak_core_entropy_info`) 26 | A helper module to get information about the entropy system, nothing needs touching here. 27 | 28 | ## Exchange FSM (`riak_core_exchange_fsm`) 29 | FSM to do the actual work of exchange. 30 | 31 | ## Index Hash Tree (`riak_core_index_hashtree`) 32 | One of those is started for each combination of service/partition, it handles the hash tree and takes care of keeping track of the changes. 33 | 34 | The VNode will need to take care of creating the hash tree for it's partition and also send updates on writes and deletes to keep the data in check. The hash tree will also fold over the entries for the first bootup. 35 | 36 | ## VNode (`riak_core_aae_vnode`) 37 | `riak_core_aae_vnode` implements a behavior for all the functions that need to be implemented for this. 38 | 39 | In addition to the functions described there the vnode needs to respond to the following calls to handle_info and command: 40 | 41 | ```erlang 42 | % ... 43 | 44 | %% Create a hahstree on init 45 | init([Idx]) -> 46 | HT = riak_core_aae_vnode:maybe_create_hashtrees(?SERVICE, Idx, undefined), 47 | {ok, #state{dict = orddict:new(), hashtrees = HT, index = Idx}}. 48 | % ... 49 | %% This is used for rehashing the tree 50 | handle_command(?FOLD_REQ{foldfun=Fun, acc0=Acc0}, _Sender, State) -> 51 | lager:debug("Fold on ~p", [State#state.partition]), 52 | Acc = orddict:fold(State#state.dict, fun(K, V, A) -> 53 | Fun({<<"bucket">>, K}, V, A) 54 | end, Acc0), 55 | {reply, Acc, State}; 56 | 57 | % ... 58 | 59 | %% retry_create_hashtree is required by `riak_core_aae_vnode:maybe_create_hashtrees` 60 | %% if the creation failed 61 | handle_info(retry_create_hashtree, State=#state{hashtrees=undefined, index=Idx}) -> 62 | lager:debug("~p/~p retrying to create a hash tree.", [?SERVICE, Idx]), 63 | HT = riak_core_aae_vnode:maybe_create_hashtrees(?SERVICE, Idx, undefined), 64 | {ok, State#state{hashtrees = HT}}; 65 | handle_info(retry_create_hashtree, State) -> 66 | {ok, State}; 67 | %% When the hashtree goes down we want to create a new one. 68 | handle_info({'DOWN', _, _, Pid, _}, State=#state{hashtrees=Pid, index=Idx}) -> 69 | lager:debug("~p/~p hashtree ~p went down.", [?SERVICE, Idx, Pid]), 70 | erlang:send_after(1000, self(), retry_create_hashtree), 71 | {ok, State#state{hashtrees = undefined}}; 72 | handle_info({'DOWN', _, _, _, _}, State) -> 73 | {ok, State}; 74 | handle_info(_, State) -> 75 | {ok, State}. 76 | 77 | %% Example write operation that uses `riak_core_aae_vnode:update_hashtree` to update 78 | %% the hashtree. 79 | do_put(Key, Obj, State) -> 80 | Dict1 = orddict:store(State#state.dict, Key, Obj), 81 | riak_core_aae_vnode:update_hashtree(<<"bucket">>, Key, term_to_binary(Obj), 82 | State#state.hashtrees), 83 | State#state{dict = Dict1}. 84 | 85 | %% Example delete operation that uses `riak_core_index_hashtree:delete` to update 86 | %% the hashtree. 87 | do_delete(Key, State) -> 88 | Dict1 = orddict:erase(State#state.dict, Key), 89 | riak_core_index_hashtree:delete({<<"bucket">>, Key}, State#state.hashtrees), 90 | State#state{dict = Dict1}. 91 | 92 | ``` 93 | 94 | ## Application (`*_app.erl`) 95 | 96 | A ets table has to be created for aae to work adding the following line to the `*_app.erl` file (or anywhere really): 97 | 98 | ```erlang 99 | riak_core_entropy_info:create_table() 100 | ``` -------------------------------------------------------------------------------- /docs/balanced.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyorai/riak_core/771add5939d9049cd6fdbcb9a86a7cecafde2e63/docs/balanced.png -------------------------------------------------------------------------------- /docs/hashtree.md: -------------------------------------------------------------------------------- 1 | `hashtree.erl` implements a fixed-sized hash tree, avoiding any need 2 | for rebalancing. The tree consists of a fixed number of on-disk 3 | `segments` and a hash tree constructed over these `segments`. Each 4 | level of the tree is grouped into buckets based on a fixed `tree 5 | width`. Each hash at level `i` corresponds to the hash of a bucket of 6 | hashes at level `i+1`. The following figure depicts a tree with 16 7 | segments and a tree-width of 4: 8 | 9 | ![image](https://github.com/basho/riak_kv/raw/jdb-hashtree/docs/hashtree.png) 10 | 11 | To insert a new `(key, hash)` pair, the key is hashed and mapped to 12 | one of the segments. The `(key, hash)` pair is then stored in the 13 | appropriate segment, which is an ordered `(key, hash)` dictionary. The 14 | given segment is then marked as dirty. Whenever `update_tree` is 15 | called, the hash for each dirty segment is re-computed, the 16 | appropriate leaf node in the hash tree updated, and the hash tree is 17 | updated bottom-up as necessary. Only paths along which hashes have 18 | been changed are re-computed. 19 | 20 | The current implementation uses LevelDB for the heavy lifting. Rather 21 | than reading/writing the on-disk segments as a unit, `(key, hash)` 22 | pairs are written to LevelDB as simple key-value pairs. The LevelDB 23 | key written is the binary `<<$s, SegmentId:64/integer, 24 | Key/binary>>`. Thus, inserting a new key-value hash is nothing more 25 | than a single LevelDB write. Likewise, key-hash pairs for a segment 26 | are laided on sequentially on-disk based on key sorting. An in-memory 27 | bitvector is used to track dirty segments, although a `gb_sets` was 28 | formerly used. 29 | 30 | When updating the segment hashes, a LevelDB iterator is used to access 31 | the segment keys in-order. The iterator seeks to the beginning of the 32 | segment and then iterators through all of the key-hash pairs. As an 33 | optimization, the iteration process is designed to read in multiple 34 | segments when possible. For example, if the list of dirty segments was 35 | `[1, 2, 3, 5, 6, 10]`, the code will seek an iterator to the beginning 36 | of segment 1, iterator through all of its keys, compute the 37 | appropriate segment 1 hash, then continue to traverse through segment 38 | 2 and segment 3's keys, updating those hashes as well. After segment 39 | 3, a new iterator will be created to seek to the beginning of segment 40 | 5, and handle both 5, and 6; and then a final iterator used to access 41 | segment 10. This design works very well when constructing a new tree 42 | from scratch. There's a phase of inserting a bunch of key-hash pairs 43 | (all writes), followed by an in-order traversal of the LevelDB 44 | database (all reads). 45 | 46 | Trees are compared using standard hash tree approach, comparing the 47 | hash at each level, and recursing to the next level down when 48 | different. After reaching the leaf nodes, any differing hashes results 49 | in a key exchange of the keys in the associated differing segments. 50 | 51 | By default, the hash tree itself is entirely in-memory. However, the 52 | code provides a `MEM_LEVEL` paramemter that specifics that levels 53 | greater than the parameter should be stored on-disk instead. These 54 | buckets are simply stored on disk in the same LevelDB structure as 55 | `{$b, Level, Bucket} -> orddict(Key, Hash)}` objects. 56 | 57 | The default settings use `1024*1024` segments with a tree width of 58 | `1024`. Thus, the resulting tree is only 3 levels deep. And there 59 | are only `1+1024+1024*1024` hashs stored in memory -- so, a few 60 | MB per hash tree. Given `1024*1024` on-disk segments, and assuming 61 | the code uniformly hashes keys to each segment, you end up with ~1000 62 | keys per segment with a 1 billion key hash tree. Thus, a single key 63 | difference would require 3 hash exchanges and a key exchange of 64 | 1000 keys to determine the differing key. 65 | -------------------------------------------------------------------------------- /docs/hashtree.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyorai/riak_core/771add5939d9049cd6fdbcb9a86a7cecafde2e63/docs/hashtree.png -------------------------------------------------------------------------------- /docs/partitioning.md: -------------------------------------------------------------------------------- 1 | # Riak's Partitioning Scheme 2 | 3 | ## Introduction 4 | 5 | When diving into the depths of Riak Core into topics such as handoff and coverage plans, it is important to understand the differences between partitions and vnodes, and how objects are placed around the cluster. 6 | 7 | This document is far from a comprehensive look at those topics, but it provides a starting point for further exploration through examples of inspecting a running Riak instance. 8 | 9 | ## Partitioning Logic Illustrated 10 | 11 | Let's say for the sake of argument that a key we store in Riak maps to a hash value just above 0. Where does it get stored and how do we represent that in our coverage plans? 12 | 13 | Our vnodes are uniquely identified via a hash value which is a multiple of the partition size in this cluster. With a ring size of 8, our vnode identifiers will be: 14 | 15 | * 0 16 | * 182687704666362864775460604089535377456991567872 17 | * 365375409332725729550921208179070754913983135744 18 | * 548063113999088594326381812268606132370974703616 19 | * 730750818665451459101842416358141509827966271488 20 | * 913438523331814323877303020447676887284957839360 21 | * 1096126227998177188652763624537212264741949407232 22 | * 1278813932664540053428224228626747642198940975104 23 | 24 | For sanity's sake, I'll simply refer to those hash values by their 25 | index along the range, so 0-7. Erlang's `bsr` operator can be used to collapse these values down to the indexes: 26 | 27 | ```erlang 28 | > 1278813932664540053428224228626747642198940975104 bsr 157. 29 | 7 30 | ``` 31 | 32 | Why 157? Because we're in a hash space size of 2^160, and with a ring size of 8 (2^3) we can throw away 157 bits (160 - 3) to isolate the key space of interest. 33 | 34 | Now let's look at where a Riak object lands. If we want to stash an object with a key of "key" in the default bucket type, bucket name "foo", we can see that the object hashes between indexes 4 and 5: 35 | 36 | ```erlang 37 | > {ok, CB} = riak_core_ring_manager:get_chash_bin(). 38 | {ok,{chashbin,8, 39 | <<0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,32,0,...>>, 40 | {'dev1@127.0.0.1'}}} 41 | 42 | > BKey = {{<<"default">>, <<"foo">>}, <<"key">>}. 43 | {{<<"default">>,<<"foo">>},<<"key">>} 44 | 45 | > DocIdx = riak_core_util:chash_key(BKey, BucketProps). 46 | <<146,24,183,162,222,0,26,238,114,48,26,19,200,76,183,84, 47 | 157,220,142,10>> 48 | 49 | 50 | > binary:decode_unsigned(DocIdx). 51 | 834063867394203420109146416465757269547655400970 52 | 53 | > binary:decode_unsigned(DocIdx) bsr 157. 54 | 4 55 | 56 | > chashbin:responsible_index(DocIdx, CB). 57 | 913438523331814323877303020447676887284957839360 58 | 59 | > chashbin:responsible_index(DocIdx, CB) bsr 157. 60 | 5 61 | ``` 62 | 63 | Thus, an object whose bucket + key map to any key space will be mapped to the partition (and vnode) responsible for the **next** key space. 64 | 65 | ## Preference Lists 66 | 67 | Once we've identified the first vnode that should store that value, we can look at the preflist (as determined by the `n_val`) to determine the other vnodes. Assuming the cluster is fully functional, this will be the next two primary vnodes. 68 | 69 | ```erlang 70 | > riak_core_apl:get_primary_apl(DocIdx, 3, riak_kv). 71 | [{{913438523331814323877303020447676887284957839360, 72 | 'dev1@127.0.0.1'}, 73 | primary}, 74 | {{1096126227998177188652763624537212264741949407232, 75 | 'dev1@127.0.0.1'}, 76 | primary}, 77 | {{1278813932664540053428224228626747642198940975104, 78 | 'dev1@127.0.0.1'}, 79 | primary}] 80 | ``` 81 | 82 | So the vnodes with index values 5, 6, and 7 will store a key that maps to keyspace index 4 if `n_val=3`. 83 | 84 | If we assume that all values are stored with 3 copies, the map of vnode index value to keyspace index values looks like this: 85 | 86 | * 0 -> [5, 6, 7] 87 | * 1 -> [6, 7, 0] 88 | * 2 -> [7, 0, 1] 89 | * 3 -> [0, 1, 2] 90 | * 4 -> [1, 2, 3] 91 | * 5 -> [2, 3, 4] 92 | * 6 -> [3, 4, 5] 93 | * 7 -> [4, 5, 6] 94 | 95 | ## Coverage Plans 96 | 97 | Coverage plans are represented differently. 98 | 99 | Coverage plans are represented internally as a list of vnode identifiers and a list of filters. 100 | 101 | The filters are vital because there's almost always going to be overlap between vnodes in a full coverage plan. 102 | 103 | For example, `riak_core_coverage_plan:find_coverage/7` for our simple cluster of ring size 8 might give us this (intermediate) coverage plan: 104 | 105 | `[{1,[0,6,7]},{3,[1,2]},{6,[3,4,5]}]` 106 | 107 | (Note that this function is kind enough to give us index values instead of hash values. Unfortunate more of the code doesn't use this convention.) 108 | 109 | This data structure says that to scan the full hash range we need to talk to vnodes 1, 3, and 6. Since 1 and 6 give us six out of our eight key spaces, we only need two out of the three key ranges stored in vnode 3. 110 | 111 | That structure gets mapped to this "real" coverage plan: 112 | 113 | ```erlang 114 | {[{182687704666362864775460604089535377456991567872, 115 | 'dev1@127.0.0.1'}, 116 | {548063113999088594326381812268606132370974703616, 117 | 'dev1@127.0.0.1'}, 118 | {1096126227998177188652763624537212264741949407232, 119 | 'dev1@127.0.0.1'}], 120 | [{548063113999088594326381812268606132370974703616, 121 | [365375409332725729550921208179070754913983135744, 122 | 548063113999088594326381812268606132370974703616]}]} 123 | ``` 124 | 125 | The last component of the coverage plan is the filter list; we drop the filters on vnodes 1 and 6, since effectively they're not filtering anything at all, but we still need vnode 3 to be filtered. 126 | 127 | Note something interesting, however. If we take the filters from that coverage plan and turn them back into index values for easier comparison: 128 | 129 | ```erlang 130 | > [X bsr 157 || X <- [365375409332725729550921208179070754913983135744, 548063113999088594326381812268606132370974703616]]. 131 | [2,3] 132 | ``` 133 | 134 | Compare that to the structure we saw earlier: `[{1,[0,6,7]},{3,[1,2]},{6,[3,4,5]}]` 135 | 136 | How did `[1,2]` become `[2,3]`? 137 | 138 | This off-by-one situation arises traces back to the fact that an object is stored in the key space above the one to which it maps. 139 | -------------------------------------------------------------------------------- /docs/unbalanced.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyorai/riak_core/771add5939d9049cd6fdbcb9a86a7cecafde2e63/docs/unbalanced.png -------------------------------------------------------------------------------- /eqc/bucket_eqc_utils.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2007-2016 Basho Technologies, Inc. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | -module(bucket_eqc_utils). 22 | 23 | %% API 24 | -export([per_test_setup/2]). 25 | 26 | 27 | per_test_setup(DefaultBucketProps, TestFun) -> 28 | try 29 | meck:new(riak_core_capability, []), 30 | meck:expect(riak_core_capability, get, 31 | fun({riak_core, bucket_types}) -> true; 32 | (X) -> meck:passthrough([X]) end), 33 | os:cmd("rm -rf ./meta_temp"), 34 | riak_core_test_util:stop_pid(whereis(riak_core_ring_events)), 35 | riak_core_test_util:stop_pid(whereis(riak_core_ring_manager)), 36 | application:set_env(riak_core, claimant_tick, 4294967295), 37 | application:set_env(riak_core, broadcast_lazy_timer, 4294967295), 38 | application:set_env(riak_core, broadcast_exchange_timer, 4294967295), 39 | application:set_env(riak_core, metadata_hashtree_timer, 4294967295), 40 | application:set_env(riak_core, cluster_name, "eqc_test"), 41 | application:set_env(riak_core, default_bucket_props, DefaultBucketProps), 42 | {ok, RingEvents} = riak_core_ring_events:start_link(), 43 | {ok, RingMgr} = riak_core_ring_manager:start_link(test), 44 | {ok, Claimant} = riak_core_claimant:start_link(), 45 | {ok, MetaMgr} = riak_core_metadata_manager:start_link([{data_dir, "./meta_temp"}]), 46 | {ok, Hashtree} = riak_core_metadata_hashtree:start_link("./meta_temp/trees"), 47 | {ok, Broadcast} = riak_core_broadcast:start_link(), 48 | 49 | Results = TestFun(), 50 | 51 | riak_core_test_util:stop_pid(Broadcast), 52 | riak_core_test_util:stop_pid(Hashtree), 53 | riak_core_test_util:stop_pid(MetaMgr), 54 | riak_core_test_util:stop_pid(Claimant), 55 | unlink(RingMgr), 56 | riak_core_ring_manager:stop(), 57 | riak_core_test_util:stop_pid(RingEvents), 58 | Results 59 | after 60 | os:cmd("rm -rf ./meta_temp"), 61 | meck:unload() 62 | end. 63 | -------------------------------------------------------------------------------- /eqc/chash_eqc.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% chash_eqc: QuickCheck tests for the chash module. 4 | %% 5 | %% Copyright (c) 2007-2011 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | %% @doc QuickCheck tests for the chash module 24 | 25 | -module(chash_eqc). 26 | 27 | -ifdef(EQC). 28 | -include_lib("eqc/include/eqc.hrl"). 29 | -include_lib("eunit/include/eunit.hrl"). 30 | 31 | -define(NOTEST, true). 32 | -define(NOASSERT, true). 33 | 34 | -define(TEST_ITERATIONS, 50). 35 | -define(QC_OUT(P), 36 | eqc:on_output(fun(Str, Args) -> io:format(user, Str, Args) end, P)). 37 | -define(RINGTOP, trunc(math:pow(2,160)-1)). % SHA-1 space 38 | 39 | -export([check/0, 40 | test/0, 41 | test/1]). 42 | 43 | %%==================================================================== 44 | %% eunit test 45 | %%==================================================================== 46 | 47 | eqc_test_() -> 48 | {inparallel, 49 | [{spawn, 50 | [{setup, 51 | fun setup/0, 52 | fun cleanup/1, 53 | [ 54 | %% Run the quickcheck tests 55 | {timeout, 60000, % timeout is in msec 56 | %% Indicate the number of test iterations for each property here 57 | ?_assertEqual(true, 58 | quickcheck(numtests(?TEST_ITERATIONS, 59 | ?QC_OUT(prop_chash_next_index())))) 60 | } 61 | ] 62 | } 63 | ] 64 | } 65 | ] 66 | }. 67 | 68 | setup() -> 69 | %% Remove the logger noise. 70 | application:load(sasl), 71 | error_logger:tty(false), 72 | %% Uncomment the following lines to send log output to files. 73 | %% application:set_env(sasl, sasl_error_logger, {file, "chash_eqc_sasl.log"}), 74 | %% error_logger:logfile({open, "chash_eqc.log"}), 75 | 76 | %% TODO: Perform any required setup 77 | ok. 78 | 79 | cleanup(_) -> 80 | %% TODO: Perform any required cleanup 81 | ok. 82 | 83 | %% ==================================================================== 84 | %% eqc property 85 | %% ==================================================================== 86 | prop_chash_next_index() -> 87 | ?FORALL( 88 | {PartitionExponent, Delta}, 89 | {g_partition_exponent(), int()}, 90 | ?TRAPEXIT( 91 | begin 92 | %% Calculate the number of paritions 93 | NumPartitions = trunc(math:pow(2, PartitionExponent)), 94 | %% Calculate the integer indexes around the ring 95 | %% for the number of partitions. 96 | Inc = ?RINGTOP div NumPartitions, 97 | Indexes = [Inc * X || X <- lists:seq(0, NumPartitions-1)], 98 | %% Create a chash tuple to use for calls to chash:successors/2 99 | %% and chash:next_index/2. 100 | %% The node value is not used and so just use the default 101 | %% localhost node value. 102 | Node = 'riak@127.0.0.1', 103 | CHash = {NumPartitions, [{Index, Node} || Index <- Indexes]}, 104 | %% For each index around the ring add Delta to 105 | %% the index value and collect the results from calling 106 | %% chash:successors/2 and chash:next_index/2 for comparison. 107 | Results = 108 | [{element( 109 | 1, 110 | hd(chash:successors(<<(((Index + Delta) + ?RINGTOP) 111 | rem ?RINGTOP):160/integer>>, 112 | CHash))), 113 | chash:next_index((((Index + Delta) + ?RINGTOP) rem ?RINGTOP), 114 | CHash)} || 115 | Index <- Indexes], 116 | {ExpectedIndexes, ActualIndexes} = lists:unzip(Results), 117 | ?WHENFAIL( 118 | begin 119 | io:format("ExpectedIndexes: ~p AcutalIndexes: ~p~n", 120 | [ExpectedIndexes, ActualIndexes]) 121 | end, 122 | conjunction( 123 | [ 124 | {results, equals(ExpectedIndexes, ActualIndexes)} 125 | ])) 126 | end 127 | )). 128 | 129 | %%==================================================================== 130 | %% Generators 131 | %%==================================================================== 132 | 133 | g_partition_exponent() -> 134 | choose(1, 12). 135 | 136 | %%==================================================================== 137 | %% Helpers 138 | %%==================================================================== 139 | 140 | test() -> 141 | test(100). 142 | 143 | test(N) -> 144 | quickcheck(numtests(N, prop_chash_next_index())). 145 | 146 | check() -> 147 | check(prop_chash_next_index(), current_counterexample()). 148 | 149 | -endif. % EQC 150 | -------------------------------------------------------------------------------- /eqc/riak_core_ring_eqc.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% 4 | %% Copyright (c) 2013 Basho Technologies, Inc. All Rights Reserved. 5 | %% 6 | %% This file is provided to you under the Apache License, 7 | %% Version 2.0 (the "License"); you may not use this file 8 | %% except in compliance with the License. You may obtain 9 | %% a copy of the License at 10 | %% 11 | %% http://www.apache.org/licenses/LICENSE-2.0 12 | %% 13 | %% Unless required by applicable law or agreed to in writing, 14 | %% software distributed under the License is distributed on an 15 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | %% KIND, either express or implied. See the License for the 17 | %% specific language governing permissions and limitations 18 | %% under the License. 19 | %% 20 | %% ------------------------------------------------------------------- 21 | 22 | -module(riak_core_ring_eqc). 23 | 24 | -ifdef(EQC). 25 | -export([prop_future_index/0]). 26 | 27 | -include_lib("eqc/include/eqc.hrl"). 28 | -include_lib("eunit/include/eunit.hrl"). 29 | 30 | -define(TEST_ITERATIONS, 10000). 31 | -define(QC_OUT(P), 32 | eqc:on_output(fun(Str, Args) -> io:format(user, Str, Args) end, P)). 33 | 34 | 35 | eqc_test_() -> 36 | {inparallel, 37 | [{spawn, 38 | [{setup, 39 | fun() -> ok end, 40 | fun(_) -> ok end, 41 | [ 42 | %% Run the quickcheck tests 43 | {timeout, 60000, % timeout is in msec 44 | %% Indicate the number of test iterations for each property here 45 | ?_assertEqual(true, 46 | quickcheck(numtests(?TEST_ITERATIONS, 47 | ?QC_OUT(prop_future_index())))) 48 | }]}]}]}. 49 | 50 | 51 | 52 | prop_future_index() -> 53 | ?FORALL({CHashKey, OrigIdx, TargetIdx, N, Pos, Ring}=TransferItem, resize_item(), 54 | ?WHENFAIL(prop_future_index_failed(TransferItem), 55 | collect(N =/= undefined andalso Pos >= N, 56 | begin 57 | {Time, Val} = timer:tc(riak_core_ring, future_index, 58 | [CHashKey, OrigIdx, N, Ring]), 59 | measure(future_index_usec, Time, TargetIdx =:= Val) 60 | end))). 61 | 62 | resize_item() -> 63 | %% RingSize - Starting Ring Size 64 | %% GrowthFactor - >1 expanding, <1 shrinking 65 | %% IndexStart - first partition in preflist for key 66 | %% Pos - position of source partition in preflist 67 | %% N - optional known N-value for the key 68 | ?LET({RingSize, GrowthF}, 69 | {current_size(), growth_factor()}, 70 | ?LET({IndexStart, Pos, N}, 71 | {index_in_current_ring(RingSize), 72 | replica_position(RingSize), 73 | check_nval(RingSize, GrowthF)}, 74 | begin 75 | Ring0 = riak_core_ring:fresh(RingSize, node()), 76 | Ring1 = riak_core_ring:resize(Ring0,trunc(GrowthF * RingSize)), 77 | Ring = riak_core_ring:set_pending_resize(Ring1, Ring0), 78 | CHashKey = <<(IndexStart-1):160/integer>>, 79 | Preflist = riak_core_ring:preflist(CHashKey, Ring0), 80 | FuturePreflist = riak_core_ring:preflist(CHashKey, Ring1), 81 | {SourceIdx, _} = lists:nth(Pos+1, Preflist), 82 | 83 | %% account for case where position is greater than 84 | %% future ring size or possbly known N-value 85 | %% (shrinking) we shouldn't have a target index in 86 | %% that case since a transfer to it would be invalid 87 | case Pos >= riak_core_ring:num_partitions(Ring1) orelse 88 | (N =/= undefined andalso Pos >= N) of 89 | true -> TargetIdx = undefined; 90 | false -> {TargetIdx, _} = lists:nth(Pos+1, FuturePreflist) 91 | end, 92 | {CHashKey, SourceIdx, TargetIdx, N, Pos, Ring} 93 | end)). 94 | 95 | index_in_current_ring(RingSize) -> 96 | elements([I || {I,_} <- riak_core_ring:all_owners(riak_core_ring:fresh(RingSize, node()))]). 97 | 98 | current_size() -> 99 | elements([16, 32, 64, 128]). 100 | 101 | growth_factor() -> 102 | oneof([0.25, 0.5, 2, 4]). 103 | 104 | 105 | replica_position(RingSize) -> 106 | %% use a max position that could be invalid for shrinking (greater than future size) 107 | %% purposefully to generate negative cases 108 | Max = RingSize, 109 | choose(0, (Max - 1)). 110 | 111 | check_nval(RingSize, GrowthF) -> 112 | case GrowthF > 1 of 113 | %% while expanding the n-value doesn't matter 114 | true -> undefined; 115 | %% while shrinking provide a "realistic" n-value of the key 116 | false -> choose(1, trunc((RingSize * GrowthF) / 2) - 1) 117 | end. 118 | 119 | 120 | prop_future_index_failed({CHashKey, OrigIdx, TargetIdx, NValCheck, _, R}) -> 121 | <> = CHashKey, 122 | FoundTarget = riak_core_ring:future_index(CHashKey, OrigIdx, NValCheck, R), 123 | io:format("key: ~p~nsource: ~p~ncurrsize: ~p~nfuturesize: ~p~nexpected: ~p~nactual: ~p~n", 124 | [CHashInt, OrigIdx, 125 | riak_core_ring:num_partitions(R), riak_core_ring:future_num_partitions(R), 126 | TargetIdx, FoundTarget]). 127 | 128 | -endif. 129 | -------------------------------------------------------------------------------- /include/riak_core.hrl: -------------------------------------------------------------------------------- 1 | -ifdef(namespaced_types). 2 | -type riak_core_dict() :: dict:dict(). 3 | -type riak_core_set() :: sets:set(). 4 | -else. 5 | -type riak_core_dict() :: dict(). 6 | -type riak_core_set() :: set(). 7 | -endif. 8 | -------------------------------------------------------------------------------- /include/riak_core_bg_manager.hrl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2013 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | -type bg_lock() :: any(). 20 | -type bg_token() :: any(). 21 | -type bg_resource() :: bg_token() | bg_lock(). 22 | -type bg_resource_type() :: lock | token. 23 | 24 | -type bg_meta() :: undefined | {atom(), any()}. %% meta data to associate with a lock/token 25 | -type bg_period() :: pos_integer(). %% token refill period in milliseconds 26 | -type bg_count() :: pos_integer(). %% token refill tokens to count at each refill period 27 | -type bg_rate() :: undefined | {bg_period(), bg_count()}. %% token refill rate 28 | -type bg_concurrency_limit() :: non_neg_integer() | infinity. %% max lock concurrency allowed 29 | -type bg_consumer() :: {undefined | pid(), bg_meta()}. %% a consumer of a resource 30 | 31 | %% Results of a "ps" of live given or blocked locks/tokens 32 | -record(bg_stat_live, 33 | { 34 | resource :: bg_resource(), %% resource name, e.g. 'aae_hashtree_lock' 35 | type :: bg_resource_type(), %% resource type, e.g. 'lock' 36 | owner :: bg_consumer() %% this consumer has the lock or token 37 | }). 38 | -type bg_stat_live() :: #bg_stat_live{}. 39 | 40 | -define(BG_INFO_ETS_TABLE, background_mgr_info_table). %% name of lock/token manager info ETS table 41 | -define(BG_INFO_ETS_OPTS, [public, %% creation time properties of info ETS table 42 | named_table, 43 | set]). 44 | 45 | -define(BG_ENTRY_ETS_TABLE, background_mgr_entry_table). %% name of lock/token manager entry ETS table 46 | -define(BG_ENTRY_ETS_OPTS, [public, %% creation time properties of entry ETS table 47 | named_table, 48 | bag]). 49 | 50 | 51 | -------------------------------------------------------------------------------- /include/riak_core_bucket_type.hrl: -------------------------------------------------------------------------------- 1 | -define(BUCKET_TYPE_PREFIX, {core, bucket_types}). 2 | -define(DEFAULT_TYPE, <<"default">>). 3 | 4 | -------------------------------------------------------------------------------- /include/riak_core_dtrace.hrl: -------------------------------------------------------------------------------- 1 | %% NOTE: Coordinate ?DTRACE_TAG_KEY with riak_core_dtrace.erl 2 | -define(DTRACE_TAG_KEY, '**DTRACE*TAG*KEY**'). 3 | 4 | %% Erlang tracing-related funnel functions for DTrace/SystemTap. When 5 | %% using Redbug or other Erlang tracing framework, trace these 6 | %% functions. Include these here to avoid copy/pasting to every 7 | %% module. 8 | 9 | dtrace(_BKey, Category, Ints, Strings) -> 10 | riak_core_dtrace:dtrace(Category, Ints, Strings). 11 | 12 | %% Internal functions, not so interesting for Erlang tracing. 13 | 14 | dtrace_int(Category, Ints, Strings) -> 15 | dtrace(get(?DTRACE_TAG_KEY), Category, Ints, Strings). 16 | -------------------------------------------------------------------------------- /include/riak_core_handoff.hrl: -------------------------------------------------------------------------------- 1 | -ifdef(namespaced_types). 2 | -type riak_core_handoff_dict() :: dict:dict(). 3 | -else. 4 | -type riak_core_handoff_dict() :: dict(). 5 | -endif. 6 | 7 | -define(PT_MSG_INIT, 0). 8 | -define(PT_MSG_OBJ, 1). 9 | -define(PT_MSG_OLDSYNC, 2). 10 | -define(PT_MSG_SYNC, 3). 11 | -define(PT_MSG_CONFIGURE, 4). 12 | -define(PT_MSG_BATCH, 5). 13 | -define(PT_MSG_VERIFY_NODE, 6). 14 | -define(PT_MSG_UNKNOWN, 255). 15 | 16 | -record(ho_stats, 17 | { 18 | interval_end :: erlang:timestamp(), 19 | last_update :: erlang:timestamp() | undefined, 20 | objs=0 :: non_neg_integer(), 21 | bytes=0 :: non_neg_integer() 22 | }). 23 | 24 | -type ho_stats() :: #ho_stats{}. 25 | -type ho_type() :: ownership | hinted | repair | resize. 26 | -type predicate() :: fun((any()) -> boolean()). 27 | 28 | -type index() :: chash:index_as_int(). 29 | -type mod_src_tgt() :: {module(), index(), index()} | {undefined, undefined, undefined}. 30 | -type mod_partition() :: {module(), index()}. 31 | 32 | -record(handoff_status, 33 | { mod_src_tgt :: mod_src_tgt(), 34 | src_node :: node(), 35 | target_node :: node(), 36 | direction :: inbound | outbound, 37 | transport_pid :: pid(), 38 | transport_mon :: reference(), 39 | timestamp :: tuple(), 40 | status :: any(), 41 | stats :: riak_core_handoff_dict(), 42 | vnode_pid :: pid() | undefined, 43 | vnode_mon :: reference() | undefined, 44 | type :: ho_type() | undefined, 45 | req_origin :: node(), 46 | filter_mod_fun :: {module(), atom()} | undefined, 47 | size :: {function(), dynamic} | {non_neg_integer(), bytes | objects} | undefined 48 | }). 49 | -type handoff_status() :: #handoff_status{}. 50 | 51 | -type known_handoff() :: {{module(), index()}, 52 | {ho_type()|'delete', 53 | 'inbound'|'outbound'|'local', 54 | node()|'$resize'|'$delete'}}. 55 | -------------------------------------------------------------------------------- /include/riak_core_metadata.hrl: -------------------------------------------------------------------------------- 1 | -type metadata_prefix() :: {binary() | atom(), binary() | atom()}. 2 | -type metadata_key() :: any(). 3 | -type metadata_pkey() :: {metadata_prefix(), metadata_key()}. 4 | -type metadata_value() :: any(). 5 | -type metadata_tombstone() :: '$deleted'. 6 | -type metadata_resolver() :: fun((metadata_value() | metadata_tombstone(), 7 | metadata_value() | metadata_tombstone()) -> metadata_value()). 8 | -type metadata_modifier() :: fun(([metadata_value() | metadata_tombstone()] | undefined) -> 9 | metadata_value()). 10 | -type metadata_object() :: {metadata, dvvset:clock()}. 11 | -type metadata_context() :: dvvset:vector(). 12 | 13 | -record(metadata_broadcast, { 14 | pkey :: metadata_pkey(), 15 | obj :: metadata_object() 16 | }). 17 | -type metadata_broadcast() :: #metadata_broadcast{}. 18 | -------------------------------------------------------------------------------- /include/riak_core_ring.hrl: -------------------------------------------------------------------------------- 1 | -define(LEGACY_RING_VSN, 1). 2 | -define(CURRENT_RING_VSN, 2). 3 | -------------------------------------------------------------------------------- /include/riak_core_vnode.hrl: -------------------------------------------------------------------------------- 1 | -type sender_type() :: fsm | server | raw. 2 | -type sender() :: {sender_type(), reference() | tuple(), pid()} | 3 | %% TODO: Double-check that these special cases are kosher 4 | {server, undefined, undefined} | % special case in 5 | % riak_core_vnode_master.erl 6 | {fsm, undefined, pid()} | % special case in 7 | % riak_kv_util:make_request/2.erl 8 | ignore. 9 | -type partition() :: chash:index_as_int(). 10 | -type vnode_req() :: term(). 11 | -type keyspaces() :: [{partition(), [partition()]}]. 12 | 13 | -record(riak_vnode_req_v1, { 14 | index :: partition() | undefined, 15 | sender=ignore :: sender(), 16 | request :: vnode_req()}). 17 | 18 | -record(riak_coverage_req_v1, { 19 | index :: partition(), 20 | keyspaces :: keyspaces(), 21 | sender=ignore :: sender(), 22 | request :: vnode_req()}). 23 | 24 | -record(riak_core_fold_req_v1, { 25 | foldfun :: fun(), 26 | acc0 :: term()}). 27 | -record(riak_core_fold_req_v2, { 28 | foldfun :: fun(), 29 | acc0 :: term(), 30 | forwardable :: boolean(), 31 | opts = [] :: list()}). 32 | 33 | -define(VNODE_REQ, #riak_vnode_req_v1). 34 | -define(COVERAGE_REQ, #riak_coverage_req_v1). 35 | -define(FOLD_REQ, #riak_core_fold_req_v2). 36 | -define(KV_VNODE_LOCK(Idx), {vnode_lock, Idx}). 37 | 38 | -type handoff_dest() :: {riak_core_handoff_manager:ho_type(), {partition(), node()}}. 39 | 40 | %% An integer, and the number of bits to shift it left to treat it as 41 | %% a mask in the 2^160 key space 42 | %% 43 | %% For a more thorough explanation of how these structures are used, 44 | %% see `riak_core_coverage_plan'. 45 | -type subpartition() :: { non_neg_integer(), pos_integer() }. 46 | 47 | -record(vnode_coverage, { 48 | vnode_identifier = 0 :: non_neg_integer(), 49 | partition_filters = [] :: [non_neg_integer()], 50 | subpartition = undefined :: undefined | subpartition() 51 | }). 52 | 53 | -type vnode_selector() :: all | allup. 54 | -type vnode_coverage() :: #vnode_coverage{}. 55 | -------------------------------------------------------------------------------- /mkcerts.sh: -------------------------------------------------------------------------------- 1 | openssl dsaparam 1024 -out params-site1.pem 2 | openssl gendsa -out test/site1-key.pem params-site1.pem 3 | openssl req -new -key test/site1-key.pem -out req-site1.pem -subj '/CN=US' 4 | openssl x509 -req -in req-site1.pem -signkey test/site1-key.pem -out test/site1-cert.pem 5 | rm req-site1.pem params-site1.pem 6 | 7 | openssl dsaparam 1024 -out params-site2.pem 8 | openssl gendsa -out test/site2-key.pem params-site2.pem 9 | openssl req -new -key test/site2-key.pem -out req-site2.pem -subj '/CN=US' 10 | openssl x509 -req -in req-site2.pem -signkey test/site2-key.pem -out test/site2-cert.pem 11 | rm req-site2.pem params-site2.pem 12 | -------------------------------------------------------------------------------- /rebar.config: -------------------------------------------------------------------------------- 1 | {erl_first_files, ["src/gen_nb_server.erl", "src/riak_core_gen_server.erl", 2 | "src/riak_core_stat_xform"]}. 3 | %{cover_enabled, true}. 4 | {erl_opts, [warnings_as_errors, 5 | {parse_transform, lager_transform}, 6 | debug_info]}. 7 | {edoc_opts, [{preprocess, true}]}. 8 | {eunit_opts, [verbose]}. 9 | {xref_checks, []}. 10 | {xref_queries, [{"(XC - UC) || (XU - X - B - \"(cluster_info|dtrace)\" : Mod)", []}]}. 11 | 12 | {deps, [ 13 | %% Stuff we need to work with rebar3 14 | {goldrush, "~>0.1.8"}, 15 | {cuttlefish, "~>2.1.4"}, 16 | {clique, "~>0.3.11"}, 17 | %% We now include folsom 18 | folsom, 19 | %% Normal deps 20 | {lager, "~>3.6.0"}, 21 | {poolboy, "0.8.4", {pkg, basho_poolboy}}, 22 | {basho_stats, "~>1.0.3"}, 23 | {riak_sysmon, "~>2.1.7"}, 24 | {riak_ensemble, "~>2.4.3", {pkg, riak_ensemble_ng}}, 25 | {pbkdf2, "~>2.0.0"}, 26 | {blume, "~>0.1.0"}, 27 | {chash, "~>0.1.1"}, 28 | {eleveldb, "~>2.2.20"}, 29 | gen_fsm_compat, 30 | {exometer_core, "~>1.0.0", {pkg, basho_exometer_core}} 31 | ]}. 32 | 33 | {plugins, [{rebar_erl_vsn, "~>0.1.10"}]}. 34 | {provider_hooks, [{pre, [ 35 | {compile, {default, erl_vsn}} 36 | ]}]}. 37 | 38 | %%------------------------------------------------------------------- 39 | %% Profiles 40 | %%------------------------------------------------------------------- 41 | 42 | {profiles, 43 | [{test, [{erl_opts, [nowarn_export_all]}, 44 | {deps, 45 | [{mustache, ".*", {git, "https://github.com/mojombo/mustache.erl.git", {tag, "v0.1.1"}}}]}]}, 46 | {docs, [{deps, [{edown, "0.7.0"}]}]}, 47 | {eqc, [{erl_opts, [nowarn_export_all, {d, 'EQC'}, {d, 'TEST'}]}, {deps, [meck]}, {plugins, [{rebar_eqc, "~>0.1.0"}]}]}, 48 | {prod, [{relx, [{dev_mode, false}]}]}, 49 | {lint, 50 | [{plugins, 51 | [{rebar3_lint, 52 | {git, "https://github.com/project-fifo/rebar3_lint.git", 53 | {tag, "0.1.4"}}}]}]} 54 | ]}. 55 | {overrides, 56 | [ 57 | %% Normal 58 | {override, setup, [{post_hooks, []}]}, 59 | {override, eleveldb, 60 | [{pre_hooks, [{compile, "c_src/build_deps.sh get-deps"}, 61 | {compile, "c_src/build_deps.sh"}]}, 62 | 63 | {post_hooks, [{clean, "c_src/build_deps.sh clean"}]}, 64 | 65 | {plugins, 66 | [pc]}, 67 | 68 | {provider_hooks, 69 | [{post, 70 | [{compile, {pc, compile}}, 71 | {clean, {pc, clean}}]}]}]}]}. 72 | -------------------------------------------------------------------------------- /rebar.lock: -------------------------------------------------------------------------------- 1 | {"1.1.0", 2 | [{<<"basho_stats">>,{pkg,<<"basho_stats">>,<<"1.0.3">>},0}, 3 | {<<"bear">>,{pkg,<<"bear">>,<<"0.8.7">>},1}, 4 | {<<"blume">>,{pkg,<<"blume">>,<<"0.1.1">>},0}, 5 | {<<"chash">>,{pkg,<<"chash">>,<<"0.1.2">>},0}, 6 | {<<"clique">>,{pkg,<<"clique">>,<<"0.3.12">>},0}, 7 | {<<"cuttlefish">>,{pkg,<<"cuttlefish">>,<<"2.1.4">>},0}, 8 | {<<"edown">>,{pkg,<<"edown">>,<<"0.8.1">>},2}, 9 | {<<"eleveldb">>,{pkg,<<"eleveldb">>,<<"2.2.20">>},0}, 10 | {<<"exometer_core">>,{pkg,<<"basho_exometer_core">>,<<"1.0.3">>},0}, 11 | {<<"folsom">>,{pkg,<<"folsom">>,<<"0.8.7">>},0}, 12 | {<<"gen_fsm_compat">>,{pkg,<<"gen_fsm_compat">>,<<"0.3.0">>},0}, 13 | {<<"getopt">>,{pkg,<<"getopt">>,<<"1.0.1">>},1}, 14 | {<<"goldrush">>,{pkg,<<"goldrush">>,<<"0.1.9">>},0}, 15 | {<<"lager">>,{pkg,<<"lager">>,<<"3.6.3">>},0}, 16 | {<<"parse_trans">>,{pkg,<<"parse_trans">>,<<"2.9.0">>},1}, 17 | {<<"pbkdf2">>,{pkg,<<"pbkdf2">>,<<"2.0.0">>},0}, 18 | {<<"poolboy">>,{pkg,<<"basho_poolboy">>,<<"0.8.4">>},0}, 19 | {<<"riak_ensemble">>,{pkg,<<"riak_ensemble_ng">>,<<"2.4.4">>},0}, 20 | {<<"riak_sysmon">>,{pkg,<<"riak_sysmon">>,<<"2.1.7">>},0}, 21 | {<<"setup">>,{pkg,<<"setup">>,<<"1.7.0">>},1}]}. 22 | [ 23 | {pkg_hash,[ 24 | {<<"basho_stats">>, <<"7E1174151509C64FCC1934120ED32295E14F84DAAE7F84926BA2C8D3700D146C">>}, 25 | {<<"bear">>, <<"16264309AE5D005D03718A5C82641FCC259C9E8F09ADEB6FD79CA4271168656F">>}, 26 | {<<"blume">>, <<"CFB4F43688690BA81C6A79F54E4678CFD5FDEDAB692F277AE740AE4A3897360D">>}, 27 | {<<"chash">>, <<"AF02484F2640C653C4B9A8557A14CA0704989DBEDB27E7CCBC442F1903A3BCA7">>}, 28 | {<<"clique">>, <<"3D8D5CA3C60787AD13562BEA18D34149BA8915AD08214EDE91D50657198B4DC3">>}, 29 | {<<"cuttlefish">>, <<"1DFF2DA7BBB67B4EF887CDA643FB112231EA08706220D34C38CB8FE25EED7F6E">>}, 30 | {<<"edown">>, <<"7333B6F6B7BBC736C263E9CEB8261667D7C8F7D92E251829F1A9FE8F24E7B694">>}, 31 | {<<"eleveldb">>, <<"1FFF63A5055BBF4BF821F797EF76065882B193F5E8095F95FCD9287187773B58">>}, 32 | {<<"exometer_core">>, <<"E801BC203558D75645529B55429C4F3D161073A4332A020570BCADD35DDCC279">>}, 33 | {<<"folsom">>, <<"A885F0AEEE4C84270954C88A55A5A473D6B2C7493E32FFDC5765412DD555A951">>}, 34 | {<<"gen_fsm_compat">>, <<"5903549F67D595F58A7101154CBE0FDD46955FBFBE40813F1E53C23A970FF5F4">>}, 35 | {<<"getopt">>, <<"C73A9FA687B217F2FF79F68A3B637711BB1936E712B521D8CE466B29CBF7808A">>}, 36 | {<<"goldrush">>, <<"F06E5D5F1277DA5C413E84D5A2924174182FB108DABB39D5EC548B27424CD106">>}, 37 | {<<"lager">>, <<"FE78951D174616273F87F0DBC3374D1430B1952E5EFC4E1C995592D30A207294">>}, 38 | {<<"parse_trans">>, <<"3F5F7B402928FB9FD200C891E635DE909045D1EFAC40CE3F924D3892898F85EB">>}, 39 | {<<"pbkdf2">>, <<"11C23279FDED5C0027AB3996CFAE77805521D7EF4BABDE2BD7EC04A9086CF499">>}, 40 | {<<"poolboy">>, <<"45C306FF1C9F6451730DD21642EDF55FA72EBD5E2FE4A38D8D8A56B8EA21A256">>}, 41 | {<<"riak_ensemble">>, <<"F9E04052F4A7FAAD20F008DFF18D34D3552513000410CE9C5941B4F7361741E8">>}, 42 | {<<"riak_sysmon">>, <<"AF420DF0F7569E1F12BCD465745164CB6189EB93F118D5CDB3F90FEB3F8BF47D">>}, 43 | {<<"setup">>, <<"15DF8E57C6DF9755E22BFB1AEF0C640BD97E9889396FDFB2A85A5536A9043674">>}]} 44 | ]. 45 | -------------------------------------------------------------------------------- /rebar3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyorai/riak_core/771add5939d9049cd6fdbcb9a86a7cecafde2e63/rebar3 -------------------------------------------------------------------------------- /src/app_helper.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riak_core: Core Riak Application 4 | %% 5 | %% Copyright (c) 2007-2012 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | -module(app_helper). 24 | 25 | -export([get_env/1, 26 | get_env/2, 27 | get_env/3, 28 | get_prop_or_env/3, 29 | get_prop_or_env/4, 30 | try_envs/1, 31 | try_envs/2]). 32 | 33 | -ifdef(TEST). 34 | -include_lib("eunit/include/eunit.hrl"). 35 | -endif. 36 | 37 | %% =================================================================== 38 | %% Public API 39 | %% =================================================================== 40 | 41 | %% @spec get_env(App :: atom()) -> [{Key :: atom(), Value :: term()}] 42 | %% @doc Retrieve all Key/Value pairs in the env for the specified app. 43 | get_env(App) -> 44 | application:get_all_env(App). 45 | 46 | %% @spec get_env(App :: atom(), Key :: atom()) -> term() 47 | %% @doc The official way to get a value from the app's env. 48 | %% Will return the 'undefined' atom if that key is unset. 49 | get_env(App, Key) -> 50 | get_env(App, Key, undefined). 51 | 52 | %% @spec get_env(App :: atom(), Key :: atom(), Default :: term()) -> term() 53 | %% @doc The official way to get a value from this application's env. 54 | %% Will return Default if that key is unset. 55 | get_env(App, Key, Default) -> 56 | case application:get_env(App, Key) of 57 | {ok, Value} -> 58 | Value; 59 | _ -> 60 | Default 61 | end. 62 | 63 | %% @doc Retrieve value for Key from Properties if it exists, otherwise 64 | %% return from the application's env. 65 | -spec get_prop_or_env(atom(), [{atom(), term()}], atom()) -> term(). 66 | get_prop_or_env(Key, Properties, App) -> 67 | get_prop_or_env(Key, Properties, App, undefined). 68 | 69 | %% @doc Return the value for Key in Properties if it exists, otherwise return 70 | %% the value from the application's env, or Default. 71 | -spec get_prop_or_env(atom(), [{atom(), term()}], atom(), term()) -> term(). 72 | get_prop_or_env(Key, Properties, App, Default) -> 73 | case proplists:get_value(Key, Properties) of 74 | undefined -> 75 | get_env(App, Key, Default); 76 | Value -> 77 | Value 78 | end. 79 | 80 | %% @doc Like `get_env' but try multiple `{App, Key}' combos before 81 | %% returning `{default, Default}'. The return value is `{App, 82 | %% Key, Value}' so that the caller may distinguish where the 83 | %% value came from. This is useful for scenarios when the config 84 | %% app/key has changed between releases and you need to check for 85 | %% both. 86 | -spec try_envs([{atom(), atom()}], term()) -> {atom(), atom(), term()}. 87 | try_envs([{App, Key}|T], Default) -> 88 | case get_env(App, Key) of 89 | undefined -> 90 | try_envs(T, Default); 91 | Value -> 92 | {App, Key, Value} 93 | end; 94 | try_envs([], Default) -> 95 | {default, Default}. 96 | 97 | try_envs(Pairs) -> 98 | try_envs(Pairs, undefined). 99 | 100 | %% =================================================================== 101 | %% EUnit tests 102 | %% =================================================================== 103 | -ifdef(TEST). 104 | 105 | app_helper_test_() -> 106 | { setup, 107 | fun setup/0, 108 | fun cleanup/1, 109 | [ 110 | fun get_prop_or_env_default_value_test_case/0, 111 | fun get_prop_or_env_undefined_value_test_case/0, 112 | fun get_prop_or_env_from_env_test_case/0, 113 | fun get_prop_or_env_from_prop_test_case/0, 114 | fun get_prop_or_env_from_prop_with_default_test_case/0, 115 | fun try_envs_test_case/0 116 | ] 117 | }. 118 | 119 | setup() -> 120 | application:set_env(bogus_app, envkeyone, value), 121 | application:set_env(bogus_app, envkeytwo, valuetwo). 122 | 123 | cleanup(_Ctx) -> 124 | ok. 125 | 126 | get_prop_or_env_default_value_test_case() -> 127 | ?assertEqual(default, get_prop_or_env(key, [], bogus, default)). 128 | 129 | get_prop_or_env_undefined_value_test_case() -> 130 | ?assertEqual(undefined, get_prop_or_env(key, [], bogus)). 131 | 132 | get_prop_or_env_from_env_test_case() -> 133 | ?assertEqual(value, get_prop_or_env(envkeyone, [], bogus_app)). 134 | 135 | get_prop_or_env_from_prop_test_case() -> 136 | Properties = [{envkeyone, propvalue}], 137 | ?assertEqual(propvalue, get_prop_or_env(envkeyone, Properties, bogus_app)). 138 | 139 | get_prop_or_env_from_prop_with_default_test_case() -> 140 | Properties = [{envkeyone, propvalue}], 141 | ?assertEqual(propvalue, get_prop_or_env(envkeyone, Properties, bogus_app, default)). 142 | 143 | try_envs_test_case() -> 144 | Val = try_envs([{noapp, nokey}, {bogus_app, envkeyone}], failed), 145 | ?assertEqual({bogus_app, envkeyone, value}, Val), 146 | Val2 = try_envs([{bogus_app, envkeytwo}, {noapp, nokey}], failed), 147 | ?assertEqual({bogus_app, envkeytwo, valuetwo}, Val2), 148 | Val3 = try_envs([{noapp, nokey}, {blah, blah}], default), 149 | ?assertEqual({default, default}, Val3). 150 | 151 | -endif. 152 | -------------------------------------------------------------------------------- /src/process_proxy.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved. 3 | %% 4 | %% This file is provided to you under the Apache License, 5 | %% Version 2.0 (the "License"); you may not use this file 6 | %% except in compliance with the License. You may obtain 7 | %% a copy of the License at 8 | %% 9 | %% http://www.apache.org/licenses/LICENSE-2.0 10 | %% 11 | %% Unless required by applicable law or agreed to in writing, 12 | %% software distributed under the License is distributed on an 13 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | %% KIND, either express or implied. See the License for the 15 | %% specific language governing permissions and limitations 16 | %% under the License. 17 | %% 18 | %% ------------------------------------------------------------------- 19 | -module(process_proxy). 20 | -export([start_link/2, init/1, stop/1]). 21 | 22 | start_link(RegName, ProxyTo) -> 23 | proc_lib:start_link(?MODULE, init, [[self(), RegName, ProxyTo]]). 24 | 25 | init([ParentPid, RegName, ProxyTo]) -> 26 | erlang:register(RegName, self()), 27 | proc_lib:init_ack(ParentPid, {ok, self()}), 28 | loop(ProxyTo). 29 | 30 | stop(Name) -> 31 | Name ! stop. 32 | 33 | loop(ProxyTo) -> 34 | receive 35 | stop -> 36 | exit(normal); 37 | M -> 38 | ProxyTo ! M, 39 | loop(ProxyTo) 40 | end. 41 | -------------------------------------------------------------------------------- /src/riak_core.app.src: -------------------------------------------------------------------------------- 1 | {application,riak_core, 2 | [{description,"Riak Core"}, 3 | {vsn,"3.1.0"}, 4 | {modules,[]}, 5 | {registered,[]}, 6 | {included_applications,[riak_ensemble,chash]}, 7 | {applications, 8 | [kernel,stdlib,gen_fsm_compat,lager,sasl,crypto,riak_sysmon,os_mon, 9 | basho_stats,eleveldb,pbkdf2,poolboy,exometer_core,clique,folsom]}, 10 | {mod,{riak_core_app,[]}}, 11 | {env, 12 | [{cluster_name,"default"}, 13 | {platform_data_dir,"data"}, 14 | {ring_creation_size,64}, 15 | {gossip_interval,60000}, 16 | {target_n_val,4}, 17 | {wants_claim_fun,{riak_core_claim,default_wants_claim}}, 18 | {choose_claim_fun,{riak_core_claim,default_choose_claim}}, 19 | {vnode_inactivity_timeout,60000}, 20 | {handoff_concurrency,2}, 21 | {disable_http_nagle,true}, 22 | {handoff_port,8099}, 23 | {handoff_ip,"0.0.0.0"}, 24 | {dist_send_buf_size,393216}, 25 | {dist_recv_buf_size,786432}, 26 | {exometer_defaults, 27 | [{['_'], 28 | histogram, 29 | [{options, 30 | [{histogram_module,exometer_slot_slide}, 31 | {keep_high,500}]}]}]}]}, 32 | {pkg_name,"riak_core_ng"}, 33 | {maintainers,["basho","Heinz N. Gies"]}, 34 | {licenses,["Apache"]}, 35 | {links,[{"Github","https://github.com/Kyorai/riak_core"}]}]}. 36 | -------------------------------------------------------------------------------- /src/riak_core_app.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riak_core: Core Riak Application 4 | %% 5 | %% Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | -module(riak_core_app). 24 | 25 | -behaviour(application). 26 | 27 | %% Application callbacks 28 | -export([start/2, stop/1]). 29 | 30 | %% =================================================================== 31 | %% Application callbacks 32 | %% =================================================================== 33 | 34 | start(_StartType, _StartArgs) -> 35 | %% Don't add our system_monitor event handler here. Instead, let 36 | %% riak_core_sysmon_minder start it, because that process can act 37 | %% on any handler crash notification, whereas we cannot. 38 | 39 | maybe_delay_start(), 40 | ok = validate_ring_state_directory_exists(), 41 | ok = safe_register_cluster_info(), 42 | ok = add_bucket_defaults(), 43 | 44 | start_riak_core_sup(). 45 | 46 | stop(_State) -> 47 | lager:info("Stopped application riak_core.\n", []), 48 | ok. 49 | 50 | maybe_delay_start() -> 51 | case application:get_env(riak_core, delayed_start) of 52 | {ok, Delay} -> 53 | lager:info("Delaying riak_core startup as requested"), 54 | timer:sleep(Delay); 55 | _ -> 56 | ok 57 | end. 58 | 59 | validate_ring_state_directory_exists() -> 60 | riak_core_util:start_app_deps(riak_core), 61 | RingStateDir = app_helper:get_env(riak_core, ring_state_dir), 62 | case filelib:ensure_dir(filename:join(RingStateDir, "dummy")) of 63 | ok -> 64 | ok; 65 | {error, RingReason} -> 66 | lager:critical( 67 | "Ring state directory ~p does not exist, " "and could not be created: ~p", 68 | [RingStateDir, lager:posix_error(RingReason)]), 69 | throw({error, invalid_ring_state_dir}) 70 | end. 71 | 72 | safe_register_cluster_info() -> 73 | %% Register our cluster_info app callback modules, with catch if 74 | %% the app is missing or packaging is broken. 75 | catch cluster_info:register_app(riak_core_cinfo_core), 76 | ok. 77 | 78 | add_bucket_defaults() -> 79 | %% add these defaults now to supplement the set that may have been 80 | %% configured in app.config 81 | riak_core_bucket:append_bucket_defaults(riak_core_bucket_type:defaults(default_type)), 82 | ok. 83 | 84 | start_riak_core_sup() -> 85 | %% Spin up the supervisor; prune ring files as necessary 86 | case riak_core_sup:start_link() of 87 | {ok, Pid} -> 88 | ok = register_applications(), 89 | ok = add_ring_event_handler(), 90 | 91 | ok = register_capabilities(), 92 | ok = init_cli_registry(), 93 | ok = riak_core_throttle:init(), 94 | 95 | riak_core_throttle:init(), 96 | 97 | {ok, Pid}; 98 | {error, Reason} -> 99 | {error, Reason} 100 | end. 101 | 102 | register_applications() -> 103 | riak_core:register(riak_core, [{stat_mod, riak_core_stat}, 104 | {permissions, [get_bucket, 105 | set_bucket, 106 | get_bucket_type, 107 | set_bucket_type]}]), 108 | ok. 109 | 110 | add_ring_event_handler() -> 111 | ok = riak_core_ring_events:add_guarded_handler(riak_core_ring_handler, []). 112 | 113 | init_cli_registry() -> 114 | riak_core_cli_registry:load_schema(), 115 | riak_core_cli_registry:register_node_finder(), 116 | riak_core_cli_registry:register_cli(), 117 | ok. 118 | 119 | register_capabilities() -> 120 | Capabilities = [[{riak_core, vnode_routing}, 121 | [proxy, legacy], 122 | legacy, 123 | {riak_core, 124 | legacy_vnode_routing, 125 | [{true, legacy}, {false, proxy}]}], 126 | [{riak_core, staged_joins}, 127 | [true, false], 128 | false], 129 | [{riak_core, resizable_ring}, 130 | [true, false], 131 | false], 132 | [{riak_core, fold_req_version}, 133 | [v2, v1], 134 | v1], 135 | [{riak_core, security}, 136 | [true, false], 137 | false], 138 | [{riak_core, bucket_types}, 139 | [true, false], 140 | false], 141 | [{riak_core, net_ticktime}, 142 | [true, false], 143 | false]], 144 | lists:foreach( 145 | fun(Capability) -> 146 | apply(riak_core_capability, register, Capability) 147 | end, 148 | Capabilities), 149 | ok. 150 | -------------------------------------------------------------------------------- /src/riak_core_base64url.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2009-2010 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | %% @doc base64url is a wrapper around the base64 module to produce 22 | %% base64-compatible encodings that are URL safe. 23 | %% The / character in normal base64 encoding is replaced with 24 | %% the _ character, and + is replaced with -. 25 | %% This replacement scheme is named "base64url" by 26 | %% http://en.wikipedia.org/wiki/Base64 27 | 28 | -module(riak_core_base64url). 29 | 30 | -export([decode/1, 31 | decode_to_string/1, 32 | encode/1, 33 | encode_to_string/1, 34 | mime_decode/1, 35 | mime_decode_to_string/1]). 36 | 37 | -spec decode(iodata()) -> binary(). 38 | decode(Base64url) -> 39 | base64:decode(urldecode(Base64url)). 40 | 41 | -spec decode_to_string(iodata()) -> string(). 42 | decode_to_string(Base64url) -> 43 | base64:decode_to_string(urldecode(Base64url)). 44 | 45 | -spec mime_decode(iodata()) -> binary(). 46 | mime_decode(Base64url) -> 47 | base64:mime_decode(urldecode(Base64url)). 48 | 49 | -spec mime_decode_to_string(iodata()) -> string(). 50 | mime_decode_to_string(Base64url) -> 51 | base64:mime_decode_to_string(urldecode(Base64url)). 52 | 53 | -spec encode(iodata()) -> binary(). 54 | encode(Data) -> 55 | urlencode(base64:encode(Data)). 56 | 57 | -spec encode_to_string(iodata()) -> string(). 58 | encode_to_string(Data) -> 59 | urlencode(base64:encode_to_string(Data)). 60 | 61 | urlencode(Base64) when is_list(Base64) -> 62 | Padded = [urlencode_digit(D) || D <- Base64], 63 | string:strip(Padded, both, $=); 64 | urlencode(Base64) when is_binary(Base64) -> 65 | Padded = << << (urlencode_digit(D)) >> || <> <= Base64 >>, 66 | binary:replace(Padded, <<"=">>, <<"">>, [global]). 67 | 68 | urldecode(Base64url) when is_list(Base64url) -> 69 | Prepad = [urldecode_digit(D) || D <- Base64url ], 70 | Padding = padding(Prepad), 71 | Prepad ++ Padding; 72 | urldecode(Base64url) when is_binary(Base64url) -> 73 | Prepad = << << (urldecode_digit(D)) >> || <> <= Base64url >>, 74 | Padding = padding(Prepad), 75 | <>. 76 | 77 | padding(Base64) when is_binary(Base64) -> 78 | case byte_size(Base64) rem 4 of 79 | 2 -> 80 | <<"==">>; 81 | 3 -> 82 | <<"=">>; 83 | _ -> 84 | <<"">> 85 | end; 86 | padding(Base64) when is_list(Base64) -> 87 | binary_to_list(padding(list_to_binary(Base64))). 88 | 89 | urlencode_digit($/) -> $_; 90 | urlencode_digit($+) -> $-; 91 | urlencode_digit(D) -> D. 92 | 93 | urldecode_digit($_) -> $/; 94 | urldecode_digit($-) -> $+; 95 | urldecode_digit(D) -> D. 96 | -------------------------------------------------------------------------------- /src/riak_core_broadcast_handler.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2013 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | -module(riak_core_broadcast_handler). 21 | 22 | %% Return a two-tuple of message id and payload from a given broadcast 23 | -callback broadcast_data(any()) -> {any(), any()}. 24 | 25 | %% Given the message id and payload, merge the message in the local state. 26 | %% If the message has already been received return `false', otherwise return `true' 27 | -callback merge(any(), any()) -> boolean(). 28 | 29 | %% Return true if the message (given the message id) has already been received. 30 | %% `false' otherwise 31 | -callback is_stale(any()) -> boolean(). 32 | 33 | %% Return the message associated with the given message id. In some cases a message 34 | %% has already been sent with information that subsumes the message associated with the given 35 | %% message id. In this case, `stale' is returned. 36 | -callback graft(any()) -> stale | {ok, any()} | {error, any()}. 37 | 38 | %% Trigger an exchange between the local handler and the handler on the given node. 39 | %% How the exchange is performed is not defined but it should be performed as a background 40 | %% process and ensure that it delivers any messages missing on either the local or remote node. 41 | %% The exchange does not need to account for messages in-flight when it is started or broadcast 42 | %% during its operation. These can be taken care of in future exchanges. 43 | -callback exchange(node()) -> {ok, pid()} | {error, term()}. 44 | -------------------------------------------------------------------------------- /src/riak_core_cinfo_core.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Riak: A lightweight, decentralized key-value store. 4 | %% 5 | %% Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | -module(riak_core_cinfo_core). 23 | 24 | -export([cluster_info_init/0, cluster_info_generator_funs/0]). 25 | 26 | %% @spec () -> term() 27 | %% @doc Required callback function for cluster_info: initialization. 28 | %% 29 | %% This function doesn't have to do anything. 30 | 31 | cluster_info_init() -> 32 | ok. 33 | 34 | %% @spec () -> list({string(), fun()}) 35 | %% @doc Required callback function for cluster_info: return list of 36 | %% {NameForReport, FunOfArity_1} tuples to generate ASCII/UTF-8 37 | %% formatted reports. 38 | 39 | cluster_info_generator_funs() -> 40 | [ 41 | {"Riak Core config files", fun config_files/1}, 42 | {"Riak Core vnode modules", fun vnode_modules/1}, 43 | {"Riak Core ring", fun get_my_ring/1}, 44 | {"Riak Core latest ring file", fun latest_ringfile/1}, 45 | {"Riak Core active partitions", fun active_partitions/1} 46 | ]. 47 | 48 | vnode_modules(CPid) -> % CPid is the data collector's pid. 49 | cluster_info:format(CPid, "~p\n", [riak_core:vnode_modules()]). 50 | 51 | get_my_ring(CPid) -> 52 | {ok, Ring} = riak_core_ring_manager:get_my_ring(), 53 | cluster_info:format(CPid, "~p\n", [Ring]). 54 | 55 | latest_ringfile(CPid) -> 56 | {ok, Path} = riak_core_ring_manager:find_latest_ringfile(), 57 | {ok, Contents} = file:read_file(Path), 58 | cluster_info:format(CPid, "Latest ringfile: ~s\n", [Path]), 59 | cluster_info:format(CPid, "File contents:\n~p\n", [binary_to_term(Contents)]). 60 | 61 | active_partitions(CPid) -> 62 | Vnodes = [{Mod, Idx} || {Mod, Idx, _Pid} <- riak_core_vnode_manager:all_vnodes()], 63 | Partitions = lists:foldl(fun({_,P}, Ps) -> 64 | ordsets:add_element(P, Ps) 65 | end, ordsets:new(), Vnodes), 66 | cluster_info:format(CPid, "~p\n", [Partitions]). 67 | 68 | config_files(C) -> 69 | {ok, [[AppPath]]} = init:get_argument(config), 70 | EtcDir = filename:dirname(AppPath), 71 | VmPath = filename:join(EtcDir, "vm.args"), 72 | [begin 73 | cluster_info:format(C, "File: ~s\n", [os:cmd("ls -l " ++ File)]), 74 | {ok, FileBin} = file:read_file(File), 75 | cluster_info:format(C, "File contents:\n~s\n", [FileBin]) 76 | end || File <- [AppPath, VmPath]]. 77 | 78 | -------------------------------------------------------------------------------- /src/riak_core_cli_registry.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2014 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | %% @doc This module tracks registrations for clique integrations. 21 | 22 | -module(riak_core_cli_registry). 23 | 24 | -define(CLI_MODULES, [ 25 | riak_core_cluster_cli, 26 | riak_core_handoff_cli 27 | ]). 28 | 29 | -export([ 30 | register_node_finder/0, 31 | register_cli/0, 32 | load_schema/0 33 | ]). 34 | 35 | -spec register_node_finder() -> true. 36 | register_node_finder() -> 37 | F = fun() -> 38 | {ok, MyRing} = riak_core_ring_manager:get_my_ring(), 39 | riak_core_ring:all_members(MyRing) 40 | end, 41 | clique:register_node_finder(F). 42 | 43 | -spec register_cli() -> ok. 44 | register_cli() -> 45 | clique:register(?CLI_MODULES). 46 | 47 | -spec load_schema() -> ok. 48 | load_schema() -> 49 | case application:get_env(riak_core, schema_dirs) of 50 | {ok, Directories} -> 51 | ok = clique_config:load_schema(Directories); 52 | _ -> 53 | ok = clique_config:load_schema([code:lib_dir()]) 54 | end. 55 | -------------------------------------------------------------------------------- /src/riak_core_dist_mon.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Disterl socket buffer size monitor 4 | %% 5 | %% Copyright (c) 2013 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | -module(riak_core_dist_mon). 24 | 25 | -export([start_link/0, set_dist_buf_sizes/2, get_riak_env_vars/0]). 26 | 27 | %% gen_server callbacks 28 | -behavior(gen_server). 29 | -export([init/1, handle_call/3, handle_cast/2, handle_info/2, 30 | terminate/2, code_change/3]). 31 | 32 | -record(state, { 33 | sndbuf :: non_neg_integer(), 34 | recbuf :: non_neg_integer() 35 | }). 36 | 37 | %% provided for testing convenience and debugging. 38 | set_dist_buf_sizes(SndBuf, RecBuf) 39 | when (is_integer(SndBuf) andalso SndBuf > 0) andalso 40 | (is_integer(RecBuf) andalso RecBuf > 0) -> 41 | gen_server:call(?MODULE, {set_dist_buf_sizes, SndBuf, RecBuf}). 42 | 43 | 44 | start_link() -> 45 | gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). 46 | 47 | init(_) -> 48 | %% register for monitor events so we can adjust buffers on 49 | %% new connections when they're started. 50 | ok = net_kernel:monitor_nodes(true, [{node_type, visible}, nodedown_reason]), 51 | {SndBuf, RecBuf} = get_riak_env_vars(), 52 | DistCtrl = erlang:system_info(dist_ctrl), 53 | %% make sure that buffers are correct on existing connections. 54 | _ = [set_port_buffers(Port, SndBuf, RecBuf) 55 | || {_Node, Port} <- DistCtrl], 56 | {ok, #state{sndbuf=SndBuf, recbuf=RecBuf}}. 57 | 58 | get_riak_env_vars() -> 59 | {ok, SndBuf} = application:get_env(riak_core, dist_send_buf_size), 60 | {ok, RecBuf} = application:get_env(riak_core, dist_recv_buf_size), 61 | {SndBuf, RecBuf}. 62 | 63 | handle_call({set_dist_buf_sizes, SndBuf, RecBuf}, _From, State) -> 64 | _ = [set_port_buffers(Port, SndBuf, RecBuf) 65 | || {_Node, Port} <- erlang:system_info(dist_ctrl)], 66 | {reply, ok, State#state{sndbuf=SndBuf, recbuf=RecBuf}}; 67 | handle_call(Msg, _From, State) -> 68 | lager:warning("unknown call message received: ~p", [Msg]), 69 | {noreply, State}. 70 | 71 | handle_cast(Msg, State) -> 72 | lager:warning("unknown cast message received: ~p", [Msg]), 73 | {noreply, State}. 74 | 75 | 76 | handle_info({nodeup, Node, _InfoList}, #state{sndbuf=SndBuf, 77 | recbuf=RecBuf} = State) -> 78 | DistCtrl = erlang:system_info(dist_ctrl), 79 | case proplists:get_value(Node, DistCtrl) of 80 | undefined -> 81 | lager:error("Could not get dist for ~p\n~p\n", [Node, DistCtrl]), 82 | {noreply, State}; 83 | Port -> 84 | ok = set_port_buffers(Port, SndBuf, RecBuf), 85 | {noreply, State} 86 | end; 87 | handle_info({nodedown, _Node, _InfoList}, State) -> 88 | %% don't think we need to do anything here 89 | {noreply, State}; 90 | handle_info(Msg, State) -> 91 | lager:warning("unknown info message received: ~p", [Msg]), 92 | {noreply, State}. 93 | 94 | terminate(_Reason, _State) -> 95 | ok. 96 | 97 | code_change(_OldVsn, State, _Extra) -> 98 | {ok, State}. 99 | 100 | %% private 101 | 102 | set_port_buffers(Port, SndBuf, RecBuf) -> 103 | inet:setopts(Port, [{sndbuf, SndBuf}, 104 | {recbuf, RecBuf}]). 105 | -------------------------------------------------------------------------------- /src/riak_core_eventhandler_guard.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riak_core_eventhandler_guard: Guard process for persistent event handlers. 4 | %% 5 | %% Copyright (c) 2007-2011 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | -module(riak_core_eventhandler_guard). 23 | -behaviour(gen_server). 24 | -export([start_link/3, start_link/4]). 25 | -export([init/1, handle_call/3, handle_cast/2, handle_info/2, 26 | terminate/2, code_change/3]). 27 | -record(state, {handlermod, handler, exitfun}). 28 | 29 | start_link(HandlerMod, Handler, Args) -> 30 | start_link(HandlerMod, Handler, Args, undefined). 31 | 32 | start_link(HandlerMod, Handler, Args, ExitFun) -> 33 | gen_server:start_link(?MODULE, [HandlerMod, Handler, Args, ExitFun], []). 34 | 35 | init([HandlerMod, Handler, Args, ExitFun]) -> 36 | ok = gen_event:add_sup_handler(HandlerMod, Handler, Args), 37 | {ok, #state{handlermod=HandlerMod, handler=Handler, exitfun=ExitFun}}. 38 | 39 | handle_call(_Request, _From, State) -> {reply, ok, State}. 40 | 41 | handle_cast(_Msg, State) -> {noreply, State}. 42 | 43 | 44 | handle_info({gen_event_EXIT, _Handler, shutdown}, State) -> 45 | {stop, normal, State}; 46 | handle_info({gen_event_EXIT, _Handler, normal}, State) -> 47 | {stop, normal, State}; 48 | handle_info({gen_event_EXIT, Handler, _Reason}, State=#state{exitfun=undefined}) -> 49 | {stop, {gen_event_EXIT, Handler}, State}; 50 | handle_info({gen_event_EXIT, Handler, Reason}, State=#state{exitfun=ExitFun}) -> 51 | ExitFun(Handler, Reason), 52 | {stop, {gen_event_EXIT, Handler}, State}; 53 | handle_info(_Info, State) -> 54 | {noreply, State}. 55 | 56 | terminate(_Reason, #state{}) -> 57 | ok. 58 | 59 | code_change(_OldVsn, State, _Extra) -> {ok, State}. 60 | 61 | -------------------------------------------------------------------------------- /src/riak_core_eventhandler_sup.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riak_core_eventhandler_sup: supervise minder processes for gen_event handlers 4 | %% 5 | %% Copyright (c) 2007-2011 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | %% @doc supervise riak_core_eventhandler_guard processes 24 | -module(riak_core_eventhandler_sup). 25 | -behaviour(supervisor). 26 | -export([start_link/0, init/1]). 27 | -export([start_guarded_handler/3, start_guarded_handler/4, stop_guarded_handler/3]). 28 | 29 | start_guarded_handler(HandlerMod, Handler, Args) -> 30 | start_guarded_handler(HandlerMod, Handler, Args, undefined). 31 | 32 | start_guarded_handler(HandlerMod, Handler, Args, ExitFun) -> 33 | case supervisor:start_child(?MODULE, handler_spec(HandlerMod, Handler, Args, ExitFun)) of 34 | {ok, _Pid} -> ok; 35 | Other -> Other 36 | end. 37 | 38 | stop_guarded_handler(HandlerMod, Handler, Args) -> 39 | case lists:member(Handler, gen_event:which_handlers(HandlerMod)) of 40 | true -> 41 | case gen_event:delete_handler(HandlerMod, Handler, Args) of 42 | {error, module_not_found} -> 43 | {error, module_not_found}; 44 | O -> 45 | Id = {HandlerMod, Handler}, 46 | ok = supervisor:terminate_child(?MODULE, Id), 47 | ok = supervisor:delete_child(?MODULE, Id), 48 | O 49 | end; 50 | false -> 51 | {error, module_not_found} 52 | end. 53 | 54 | handler_spec(HandlerMod, Handler, Args, ExitFun) -> 55 | {{HandlerMod, Handler}, 56 | {riak_core_eventhandler_guard, start_link, [HandlerMod, Handler, Args, ExitFun]}, 57 | transient, 5000, worker, [riak_core_eventhandler_guard]}. 58 | 59 | start_link() -> 60 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 61 | 62 | %% @private 63 | init([]) -> 64 | {ok, {{one_for_one, 10, 10}, []}}. 65 | 66 | 67 | -------------------------------------------------------------------------------- /src/riak_core_exo_monitor.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2014 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This Source Code Form is subject to the terms of the Mozilla Public 6 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 7 | %% file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | %% 9 | %% ------------------------------------------------------------------- 10 | %% 11 | %% @doc Legacy mapping module folsom metrics to exometer 12 | %% 13 | %% @end 14 | -module(riak_core_exo_monitor). 15 | -behaviour(exometer_folsom_monitor). 16 | -behaviour(exometer_entry). 17 | 18 | -export([copy_folsom/3]). 19 | -export([behaviour/0, 20 | delete/3, 21 | get_datapoints/3, 22 | get_value/4, 23 | new/3, 24 | reset/3, 25 | sample/3, 26 | setopts/3, 27 | update/4]). 28 | 29 | behaviour() -> 30 | entry. 31 | 32 | copy_folsom(Name, Type, Opts) when is_tuple(Name) -> 33 | Prefix = riak_core_stat:prefix(), 34 | {[Prefix|tuple_to_list(Name)], ad_hoc, [{folsom_name, Name}, 35 | {module, ?MODULE}, 36 | {type, Type} 37 | | options(Type, Opts)]}; 38 | copy_folsom(_, _, _) -> 39 | false. 40 | 41 | new(N, _, Opts) -> 42 | {ok, {proplists:get_value(type, Opts, unknown), 43 | proplists:get_value(folsom_name, Opts, N)}}. 44 | 45 | update(_, Value, counter, {_, Name}) -> 46 | folsom_metrics:notify_existing_metric(Name, {inc,Value}, counter); 47 | update(_, Value, Type, {_, Name}) -> 48 | folsom_metrics:notify_existing_metric(Name, Value, Type). 49 | 50 | reset(_, _, _) -> 51 | {error, unsupported}. 52 | 53 | get_value(_, Type, {_, Name}, DPs) -> 54 | exometer_folsom:get_value(Name, Type, [], DPs). 55 | 56 | sample(_, _, _) -> 57 | {error, unsupported}. 58 | 59 | setopts(_, _, _) -> 60 | ok. 61 | 62 | delete(_, _, _) -> 63 | {error, unsupported}. 64 | 65 | get_datapoints(Name, Type, _) -> 66 | exometer_folsom:get_datapoints(Name, Type, []). 67 | 68 | options(history, [Size]) -> 69 | [{size, Size}]; 70 | options(histogram, [SampleType, SampleSize, Alpha]) -> 71 | [{sample_type, SampleType}, 72 | {sample_size, SampleSize}, 73 | {alpha, Alpha}]; 74 | options(duration , [SampleType, SampleSize, Alpha]) -> 75 | [{sample_type, SampleType}, 76 | {sample_size, SampleSize}, 77 | {alpha, Alpha}]; 78 | options(meter_reader, []) -> []; 79 | options(spiral , []) -> []; 80 | options(meter , []) -> []; 81 | options(gauge , []) -> []; 82 | options(counter , []) -> []. 83 | -------------------------------------------------------------------------------- /src/riak_core_format.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2007-2012 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | %% @doc Functions for formatting data. 22 | 23 | -module(riak_core_format). 24 | -export([fmt/2, 25 | human_size_fmt/2, 26 | human_time_fmt/2, 27 | epoch_to_datetime/1]). 28 | 29 | -ifdef(TEST). 30 | -include_lib("eunit/include/eunit.hrl"). 31 | -endif. 32 | 33 | %% @doc Created a string `Str' based on the format string `FmtStr' and 34 | %% list of args `Args'. 35 | -spec fmt(string(), list()) -> Str::string(). 36 | fmt(FmtStr, Args) -> 37 | lists:flatten(io_lib:format(FmtStr, Args)). 38 | 39 | %% @doc Create a human friendly string `Str' for number of bytes 40 | %% `Bytes' and format based on format string `Fmt'. 41 | -spec human_size_fmt(string(), number()) -> string(). 42 | human_size_fmt(Fmt, Bytes) -> 43 | Fmt2 = Fmt ++ " ~s", 44 | {Value, Units} = human_size(Bytes, ["B","KB","MB","GB","TB","PB"]), 45 | fmt(Fmt2, [Value, Units]). 46 | 47 | %% @doc Create a human friendly string `Str' for the given time in 48 | %% microseconds `Micros'. Format according to format string 49 | %% `Fmt'. 50 | -spec human_time_fmt(string(), non_neg_integer()) -> Str::string(). 51 | human_time_fmt(Fmt, Micros) -> 52 | Fmt2 = Fmt ++ " ~s", 53 | {Value, Units} = human_time(Micros), 54 | fmt(Fmt2, [Value, Units]). 55 | 56 | %% @doc Convert a folsom_utils:now_epoch() to a universal datetime 57 | -spec epoch_to_datetime(non_neg_integer()) -> calendar:datetime(). 58 | epoch_to_datetime(S) -> 59 | Epoch = {{1970,1,1},{0,0,0}}, 60 | Seconds = calendar:datetime_to_gregorian_seconds(Epoch) + S, 61 | calendar:gregorian_seconds_to_datetime(Seconds). 62 | 63 | %%%=================================================================== 64 | %%% Private 65 | %%%=================================================================== 66 | 67 | %% @private 68 | %% 69 | %% @doc Formats a byte size into a human-readable size with units. 70 | %% Thanks StackOverflow: 71 | %% http://stackoverflow.com/questions/2163691/simpler-way-to-format-bytesize-in-a-human-readable-way 72 | -spec human_size(number(), list()) -> {float(), iolist()}. 73 | human_size(S, [_|[_|_] = L]) when S >= 1024 -> human_size(S/1024, L); 74 | human_size(S, [M|_]) -> 75 | {float(S), M}. 76 | 77 | %% @private 78 | %% 79 | %% @doc Given a number of `Micros' returns a human friendly time 80 | %% duration in the form of `{Value, Units}'. 81 | -spec human_time(non_neg_integer()) -> {Value::number(), Units::string()}. 82 | human_time(Micros) -> 83 | human_time(Micros, {1000, "us"}, [{1000, "ms"}, {60, "s"}, {60, "min"}, {24, "hr"}, {365, "d"}]). 84 | 85 | -spec human_time(number(), {pos_integer(), string()}, 86 | [{pos_integer(), string()}]) -> 87 | {float(), string()}. 88 | human_time(T, {Divisor, Unit}, Units) when T < Divisor orelse Units == [] -> 89 | {float(T), Unit}; 90 | human_time(T, {Divisor, _}, [Next|Units]) -> 91 | human_time(T / Divisor, Next, Units). 92 | 93 | -ifdef(TEST). 94 | human_time_fmt_test() -> 95 | FiveUS = 5, 96 | FiveMS = 5000, 97 | FiveS = 5000000, 98 | FiveMin = FiveS * 60, 99 | FiveHr = FiveMin * 60, 100 | FiveDay = FiveHr * 24, 101 | [?assertEqual(Expect, human_time_fmt("~.1f", T)) 102 | || {T, Expect} <- [{FiveUS, "5.0 us"}, 103 | {FiveMS, "5.0 ms"}, 104 | {FiveS, "5.0 s"}, 105 | {FiveMin, "5.0 min"}, 106 | {FiveHr, "5.0 hr"}, 107 | {FiveDay, "5.0 d"}]]. 108 | -endif. 109 | -------------------------------------------------------------------------------- /src/riak_core_handoff_listener.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riak_handoff_listener: entry point for TCP-based handoff 4 | %% 5 | %% Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | %% @doc entry point for TCP-based handoff 24 | 25 | -module(riak_core_handoff_listener). 26 | -behavior(gen_nb_server). 27 | -export([start_link/0]). 28 | -export([init/1, handle_call/3, handle_cast/2, handle_info/2, 29 | terminate/2, code_change/3]). 30 | -export([get_handoff_ip/0, sock_opts/0, new_connection/2]). 31 | -record(state, { 32 | ipaddr :: string(), 33 | portnum :: integer(), 34 | ssl_opts :: list() 35 | }). 36 | 37 | start_link() -> 38 | PortNum = app_helper:get_env(riak_core, handoff_port), 39 | IpAddr = app_helper:get_env(riak_core, handoff_ip), 40 | SslOpts = riak_core_handoff_sender:get_handoff_ssl_options(), 41 | gen_nb_server:start_link(?MODULE, IpAddr, PortNum, [IpAddr, PortNum, SslOpts]). 42 | 43 | get_handoff_ip() -> 44 | riak_core_gen_server:call(?MODULE, handoff_ip, infinity). 45 | 46 | init([IpAddr, PortNum, SslOpts]) -> 47 | register(?MODULE, self()), 48 | 49 | %% This exit() call shouldn't be necessary, AFAICT the VM's EXIT 50 | %% propagation via linking should do the right thing, but BZ 823 51 | %% suggests otherwise. However, the exit() call should fall into 52 | %% the "shouldn't hurt", especially since the next line will 53 | %% explicitly try to spawn a new proc that will try to register 54 | %% the riak_kv_handoff_listener name. 55 | catch exit(whereis(riak_kv_handoff_listener), kill), 56 | process_proxy:start_link(riak_kv_handoff_listener, ?MODULE), 57 | 58 | {ok, #state{portnum=PortNum, ipaddr=IpAddr, ssl_opts = SslOpts}}. 59 | 60 | sock_opts() -> [binary, {packet, 4}, {reuseaddr, true}, {backlog, 64}]. 61 | 62 | handle_call(handoff_ip, _From, State=#state{ipaddr=I}) -> 63 | {reply, {ok, I}, State}; 64 | 65 | handle_call(handoff_port, _From, State=#state{portnum=P}) -> 66 | {reply, {ok, P}, State}. 67 | 68 | handle_cast(_Msg, State) -> {noreply, State}. 69 | 70 | handle_info(_Info, State) -> {noreply, State}. 71 | 72 | terminate(_Reason, _State) -> ok. 73 | 74 | code_change(_OldVsn, State, _Extra) -> {ok, State}. 75 | 76 | new_connection(Socket, State = #state{ssl_opts = SslOpts}) -> 77 | case riak_core_handoff_manager:add_inbound(SslOpts) of 78 | {ok, Pid} -> 79 | ok = gen_tcp:controlling_process(Socket, Pid), 80 | ok = riak_core_handoff_receiver:set_socket(Pid, Socket), 81 | {ok, State}; 82 | {error, _Reason} -> 83 | riak_core_stat:update(rejected_handoffs), 84 | gen_tcp:close(Socket), 85 | {ok, State} 86 | end. 87 | 88 | -------------------------------------------------------------------------------- /src/riak_core_handoff_listener_sup.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2011 Basho Technologies, Inc. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | -module(riak_core_handoff_listener_sup). 22 | -behaviour(supervisor). 23 | 24 | %% beahvior functions 25 | -export([start_link/0, 26 | init/1 27 | ]). 28 | 29 | -define(CHILD(I,Type), {I,{I,start_link,[]},permanent,brutal_kill,Type,[I]}). 30 | 31 | %% begins the supervisor, init/1 will be called 32 | start_link () -> 33 | supervisor:start_link({local,?MODULE},?MODULE,[]). 34 | 35 | %% @private 36 | init ([]) -> 37 | {ok,{{one_for_one,10,10}, 38 | [?CHILD(riak_core_handoff_listener,worker) 39 | ]}}. 40 | -------------------------------------------------------------------------------- /src/riak_core_handoff_receiver_sup.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2011 Basho Technologies, Inc. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | -module(riak_core_handoff_receiver_sup). 22 | -behaviour(supervisor). 23 | 24 | %% beahvior functions 25 | -export([start_link/0, 26 | init/1 27 | ]). 28 | 29 | %% public functions 30 | -export([start_receiver/1 31 | ]). 32 | 33 | -define(CHILD(I,Type), {I,{I,start_link,[]},temporary,brutal_kill,Type,[I]}). 34 | 35 | %% begins the supervisor, init/1 will be called 36 | start_link () -> 37 | supervisor:start_link({local,?MODULE},?MODULE,[]). 38 | 39 | %% @private 40 | init ([]) -> 41 | {ok,{{simple_one_for_one,10,10}, 42 | [?CHILD(riak_core_handoff_receiver,worker) 43 | ]}}. 44 | 45 | %% start a sender process 46 | start_receiver (SSLOpts) -> 47 | supervisor:start_child(?MODULE,[SSLOpts]). 48 | -------------------------------------------------------------------------------- /src/riak_core_handoff_sender_sup.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2011-2012 Basho Technologies, Inc. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | -module(riak_core_handoff_sender_sup). 22 | -behaviour(supervisor). 23 | 24 | %% callbacks 25 | -export([start_link/0, 26 | init/1 27 | ]). 28 | 29 | %% API 30 | -export([start_sender/5]). 31 | 32 | -include("riak_core_handoff.hrl"). 33 | -define(CHILD(I,Type), {I,{I,start_link,[]},temporary,brutal_kill,Type,[I]}). 34 | 35 | %%%=================================================================== 36 | %%% API 37 | %%%=================================================================== 38 | 39 | start_link() -> 40 | supervisor:start_link({local,?MODULE},?MODULE,[]). 41 | 42 | %% @doc Start the handoff process for the module (`Module'), partition 43 | %% (`Partition'), and vnode (`VNode') from the local node to the 44 | %% target node (`TargetNode') with options `Opts'. 45 | %% 46 | %% Options: 47 | %% * src_partition - required. the integer index of the source vnode 48 | %% * target_partition - required. the integer index of the target vnode 49 | %% * filter - optional. an arity one function that takes the key and returns 50 | %% a boolean. If false, the key is not sent 51 | %% * unsent_fun - optional. an arity 2 function that takes a key that was not sent 52 | %% (based on filter) and an accumulator. This function is called 53 | %% for each unsent key. 54 | %% * unsent_acc0 - optional. The intial accumulator value passed to unsent_fun 55 | %% for the first unsent key 56 | -spec start_sender(ho_type(), atom(), term(), pid(), [{atom(), term()}]) -> {ok, pid()}. 57 | start_sender(Type, Module, TargetNode, VNode, Opts) -> 58 | supervisor:start_child(?MODULE, [TargetNode, Module, {Type, Opts}, VNode]). 59 | 60 | %%%=================================================================== 61 | %%% Callbacks 62 | %%%=================================================================== 63 | 64 | %% @private 65 | init ([]) -> 66 | {ok,{{simple_one_for_one,10,10}, 67 | [?CHILD(riak_core_handoff_sender,worker) 68 | ]}}. 69 | -------------------------------------------------------------------------------- /src/riak_core_handoff_sup.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2011 Basho Technologies, Inc. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | -module(riak_core_handoff_sup). 22 | -behaviour(supervisor). 23 | 24 | %% beahvior functions 25 | -export([start_link/0, 26 | init/1 27 | ]). 28 | 29 | -define(CHILD(I,Type), {I,{I,start_link,[]},permanent,brutal_kill,Type,[I]}). 30 | 31 | %% begins the supervisor, init/1 will be called 32 | start_link () -> 33 | supervisor:start_link({local,?MODULE},?MODULE,[]). 34 | 35 | %% @private 36 | init ([]) -> 37 | {ok,{{one_for_all,10,10}, 38 | [?CHILD(riak_core_handoff_receiver_sup,supervisor), 39 | ?CHILD(riak_core_handoff_sender_sup,supervisor), 40 | ?CHILD(riak_core_handoff_listener_sup,supervisor), 41 | ?CHILD(riak_core_handoff_manager,worker) 42 | ]}}. 43 | -------------------------------------------------------------------------------- /src/riak_core_metadata_evt_sup.erl: -------------------------------------------------------------------------------- 1 | %%% =================================================================== 2 | %%% 3 | %%% riak_core_metadata_evt_sup supervises all event managers for 4 | %%% metadata events. One event manager is created per prefix 5 | %%% that is subscribed to. Events that are not subscribed to do 6 | %%% nothing. 7 | %%% 8 | %%% Copyright (c) 2015 Basho Technologies, Inc. All Rights Reserved. 9 | %%% 10 | %%% This file is provided to you under the Apache License, 11 | %%% Version 2.0 (the "License"); you may not use this file 12 | %%% except in compliance with the License. You may obtain 13 | %%% a copy of the License at 14 | %%% 15 | %%% http://www.apache.org/licenses/LICENSE-2.0 16 | %%% 17 | %%% Unless required by applicable law or agreed to in writing, 18 | %%% software distributed under the License is distributed on an 19 | %%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 20 | %%% KIND, either express or implied. See the License for the 21 | %%% specific language governing permissions and limitations 22 | %%% under the License. 23 | %%% 24 | %%% =================================================================== 25 | 26 | -module(riak_core_metadata_evt_sup). 27 | -behaviour(supervisor). 28 | 29 | -export([init/1]). 30 | -export([is_type_compiled/2]). 31 | -export([start_link/0]). 32 | -export([swap_notification_handler/3]). 33 | -export([sync_notify/2]). 34 | 35 | -include("riak_core_bucket_type.hrl"). 36 | 37 | -define(TABLE, ?MODULE). 38 | -define(CHILD_ID, riak_core_metadata_evt). 39 | 40 | %%% =================================================================== 41 | %%% API 42 | %%% =================================================================== 43 | 44 | %% 45 | start_link() -> 46 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 47 | 48 | %% Swap the existing `Handler' with the given one, or create a new one 49 | %% if one does not exist. Prevents event handlers being added multiple 50 | %% times. 51 | swap_notification_handler(FullPrefix, Handler, HandlerArgs) -> 52 | case ets:lookup(?TABLE, FullPrefix) of 53 | [{FullPrefix, Pid}] -> 54 | swap_notification_handler2(Pid, Handler, HandlerArgs); 55 | [] -> 56 | {ok, Pid} = supervisor:start_child(?MODULE, []) , 57 | true = ets:insert_new(?TABLE, {FullPrefix, Pid}), 58 | swap_notification_handler2(Pid, Handler, HandlerArgs) 59 | end. 60 | 61 | %% Notify all listeners that metadata has been stored, this could mean 62 | %% that a new bucket type has been created or that it has been updated. 63 | %% Listeners receive the event type `{metadata_stored, Key::any()}'. 64 | sync_notify(FullPrefix, Key) -> 65 | case (catch ets:lookup(?TABLE, FullPrefix)) of 66 | [{FullPrefix, Pid}] -> 67 | gen_event:sync_notify(Pid, {metadata_stored, Key}); 68 | _ -> 69 | ok 70 | end. 71 | 72 | %% Make a call to bucket type listeners if the bucket type ddl has been 73 | %% compiled yet, if any of the listeners returns true then it has. Listeners 74 | %% that do not handle this should return false, or any value that is not true. 75 | %% 76 | %% This should only be called if the bucket type has the ddl property. 77 | -spec is_type_compiled(BucketType :: binary(), DDL :: term()) -> boolean(). 78 | is_type_compiled(BucketType, DDL) when is_binary(BucketType) -> 79 | case ets:lookup(?TABLE, ?BUCKET_TYPE_PREFIX) of 80 | [{?BUCKET_TYPE_PREFIX, Pid}] -> 81 | Handlers = gen_event:which_handlers(Pid), 82 | Req = {is_type_compiled, [BucketType, DDL]}, 83 | Results = [gen_event:call(Pid, H, Req) || H <- Handlers], 84 | lists:member(true, Results); 85 | [] -> 86 | false 87 | end. 88 | 89 | %%% =================================================================== 90 | %%% Internal functions 91 | %%% =================================================================== 92 | 93 | %% 94 | init([]) -> 95 | % an ets table is used to map prefixes to the event manager process 96 | ?TABLE = ets:new(?TABLE, [public, named_table]), 97 | 98 | Procs = [{?CHILD_ID, 99 | {gen_event, start_link, []}, 100 | permanent, 1000, worker, []} ], 101 | {ok, {{simple_one_for_one, 1, 5}, Procs}}. 102 | 103 | %% 104 | swap_notification_handler2(Pid, Handler, HandlerArgs) when is_pid(Pid) -> 105 | Terminate_args = [], 106 | gen_event:swap_handler( 107 | Pid, {Handler, Terminate_args}, {Handler, HandlerArgs}). 108 | 109 | 110 | %% =================================================================== 111 | %% EUnit tests 112 | %% =================================================================== 113 | 114 | -ifdef(TEST). 115 | -include_lib("eunit/include/eunit.hrl"). 116 | 117 | -define(in_process(TestCode), 118 | Self = self(), 119 | spawn_link( 120 | fun() -> 121 | TestCode, 122 | Self ! test_ok 123 | end), 124 | receive 125 | test_ok -> ok 126 | end 127 | ). 128 | 129 | start_link_test() -> 130 | ?in_process( 131 | begin 132 | ?assertMatch( 133 | {ok, _}, 134 | ?MODULE:start_link() 135 | ) 136 | end). 137 | 138 | 139 | swap_notification_handler_test() -> 140 | ?in_process( 141 | begin 142 | Metadata_type = {core,bucket_types}, 143 | {ok, _} = riak_core_metadata_evt_sup:start_link(), 144 | ok = riak_core_metadata_evt_sup:swap_notification_handler( 145 | Metadata_type, dummy_evt, []), 146 | ?assertMatch( 147 | [{Metadata_type, _}], 148 | ets:tab2list(?TABLE) 149 | ) 150 | end). 151 | 152 | -endif. 153 | -------------------------------------------------------------------------------- /src/riak_core_mochiglobal.erl: -------------------------------------------------------------------------------- 1 | %% @author Bob Ippolito 2 | %% @copyright 2010 Mochi Media, Inc. 3 | %% @doc Abuse module constant pools as a "read-only shared heap" (since erts 5.6) 4 | %% [1]. 5 | -module(riak_core_mochiglobal). 6 | -author("Bob Ippolito "). 7 | -export([get/1, get/2, put/2, delete/1]). 8 | 9 | -spec get(atom()) -> any() | undefined. 10 | %% @equiv get(K, undefined) 11 | get(K) -> 12 | get(K, undefined). 13 | 14 | -spec get(atom(), T) -> any() | T. 15 | %% @doc Get the term for K or return Default. 16 | get(K, Default) -> 17 | get(K, Default, key_to_module(K)). 18 | 19 | get(_K, Default, Mod) -> 20 | try Mod:term() 21 | catch error:undef -> 22 | Default 23 | end. 24 | 25 | -spec put(atom(), any()) -> ok. 26 | %% @doc Store term V at K, replaces an existing term if present. 27 | put(K, V) -> 28 | put(K, V, key_to_module(K)). 29 | 30 | put(_K, V, Mod) -> 31 | Bin = compile(Mod, V), 32 | code:purge(Mod), 33 | {module, Mod} = code:load_binary(Mod, atom_to_list(Mod) ++ ".erl", Bin), 34 | ok. 35 | 36 | -spec delete(atom()) -> boolean(). 37 | %% @doc Delete term stored at K, no-op if non-existent. 38 | delete(K) -> 39 | delete(K, key_to_module(K)). 40 | 41 | delete(_K, Mod) -> 42 | code:purge(Mod), 43 | code:delete(Mod). 44 | 45 | -spec key_to_module(atom()) -> atom(). 46 | key_to_module(K) -> 47 | list_to_atom("mochiglobal:" ++ atom_to_list(K)). 48 | 49 | -spec compile(atom(), any()) -> binary(). 50 | compile(Module, T) -> 51 | {ok, Module, Bin} = compile:forms(forms(Module, T), 52 | [verbose, report_errors]), 53 | Bin. 54 | 55 | -spec forms(atom(), any()) -> [erl_syntax:syntaxTree()]. 56 | forms(Module, T) -> 57 | [erl_syntax:revert(X) || X <- term_to_abstract(Module, term, T)]. 58 | 59 | -spec term_to_abstract(atom(), atom(), any()) -> [erl_syntax:syntaxTree()]. 60 | term_to_abstract(Module, Getter, T) -> 61 | [%% -module(Module). 62 | erl_syntax:attribute( 63 | erl_syntax:atom(module), 64 | [erl_syntax:atom(Module)]), 65 | %% -export([Getter/0]). 66 | erl_syntax:attribute( 67 | erl_syntax:atom(export), 68 | [erl_syntax:list( 69 | [erl_syntax:arity_qualifier( 70 | erl_syntax:atom(Getter), 71 | erl_syntax:integer(0))])]), 72 | %% Getter() -> T. 73 | erl_syntax:function( 74 | erl_syntax:atom(Getter), 75 | [erl_syntax:clause([], none, [erl_syntax:abstract(T)])])]. 76 | 77 | %% 78 | %% Tests 79 | %% 80 | -ifdef(TEST). 81 | -include_lib("eunit/include/eunit.hrl"). 82 | get_put_delete_test() -> 83 | K = '$$test$$mochiglobal', 84 | delete(K), 85 | ?assertEqual( 86 | bar, 87 | get(K, bar)), 88 | try 89 | ?MODULE:put(K, baz), 90 | ?assertEqual( 91 | baz, 92 | get(K, bar)), 93 | ?MODULE:put(K, wibble), 94 | ?assertEqual( 95 | wibble, 96 | ?MODULE:get(K)) 97 | after 98 | delete(K) 99 | end, 100 | ?assertEqual( 101 | bar, 102 | get(K, bar)), 103 | ?assertEqual( 104 | undefined, 105 | ?MODULE:get(K)), 106 | ok. 107 | -endif. 108 | -------------------------------------------------------------------------------- /src/riak_core_net_ticktime.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riak_core: Core Riak Application 4 | %% 5 | %% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | %% @doc Change net_kernel's ticktime on-the-fly. 24 | 25 | -module(riak_core_net_ticktime). 26 | -export([enable/0, 27 | start_set_net_ticktime_daemon/2, 28 | stop_set_net_ticktime_daemon/1]). 29 | 30 | -define(REGNAME, net_kernel_net_ticktime_change_daemon). 31 | 32 | -spec enable() -> ok. 33 | enable() -> 34 | riak_core_capability:register({riak_core, net_ticktime}, 35 | [true, false], 36 | false). 37 | 38 | start_set_net_ticktime_daemon(Node, Time) -> 39 | start_set_net_ticktime_daemon(Node, Time, net_ticktime_active()). 40 | 41 | start_set_net_ticktime_daemon(Node, Time, true) -> 42 | EbinDir = filename:dirname(code:which(?MODULE)), 43 | try 44 | Dirs = rpc:call(Node, code, get_path, []), 45 | case lists:member(EbinDir, Dirs) of 46 | false -> 47 | lager:info("start_set_net_ticktime_daemon: adding to code path " 48 | "for node ~p\n", [Node]), 49 | rpc:call(Node, code, add_pathz, [EbinDir]); 50 | true -> 51 | ok 52 | end 53 | catch _:_ -> 54 | %% Network problems or timeouts here, spawn will fail, no 55 | %% worries, we'll try again soon. 56 | ok 57 | end, 58 | spawn(Node, fun() -> 59 | try 60 | register(?REGNAME, self()), 61 | %% If we get here, we are the one daemon process 62 | lager:info("start_set_net_ticktime_daemon: started " 63 | "changing net_ticktime on ~p to ~p\n", 64 | [Node, Time]), 65 | _ = riak_core_rand:seed(os:timestamp()), 66 | set_net_ticktime_daemon_loop(Time, 1) 67 | catch _:_ -> 68 | ok 69 | end 70 | end); 71 | start_set_net_ticktime_daemon(Node, _Time, false) -> 72 | lager:info("Not starting tick daemon on ~p. Capability unsupported. " 73 | "Some nodes in the Riak cluster do not have ~p loaded\n", 74 | [Node, ?MODULE]), 75 | ok. 76 | 77 | stop_set_net_ticktime_daemon(Node) -> 78 | Capability = riak_core_capability:get({riak_core, net_ticktime}), 79 | stop_set_net_ticktime_daemon(Node, Capability). 80 | 81 | stop_set_net_ticktime_daemon(Node, true) -> 82 | try 83 | case rpc:call(Node, erlang, whereis, [?REGNAME]) of 84 | Pid when is_pid(Pid) -> 85 | io:format("Stopping tick daemon ~p on ~p\n", [Pid, Node]), 86 | exit(Pid, stop_now), 87 | ok; 88 | undefined -> 89 | io:format("Stopping tick daemon on ~p but not running\n", [Node]), 90 | ok 91 | end 92 | catch _:_ -> 93 | %% Network problems or timeouts, we don't try too hard 94 | error 95 | end; 96 | stop_set_net_ticktime_daemon(Node, false) -> 97 | lager:info("Not stopping tick daemon on ~p. Capability unsupported\n", [Node]), 98 | ok. 99 | 100 | async_start_set_net_ticktime_daemons(Time, Nodes) -> 101 | Pids = [spawn(fun() -> 102 | start_set_net_ticktime_daemon(Node, Time, true) 103 | end) || Node <- Nodes], 104 | spawn(fun() -> 105 | %% If a daemon cannot finish in 5 seconds, no worries. 106 | %% We want to avoid leaving lots of pids around due to 107 | %% network/net_kernel instability. 108 | timer:sleep(5000), 109 | [exit(Pid, kill) || Pid <- Pids] 110 | end). 111 | 112 | set_net_ticktime_daemon_loop(Time, Count) -> 113 | case set_net_ticktime(Time) of 114 | unchanged -> 115 | lager:info("start_set_net_ticktime_daemon: finished " 116 | "changing net_ticktime on ~p to ~p\n", [node(), Time]), 117 | exit(normal); 118 | _ -> 119 | timer:sleep(riak_core_rand:uniform(1*1000)), 120 | %% Here is an uncommon use the erlang:nodes/1 BIF. 121 | %% Hidden nodes (e.g. administrative escripts) may have 122 | %% connected to us. Force them to change their tick time, 123 | %% in case they're using something different. And pick up 124 | %% any regular nodes that have connected since we started. 125 | if 126 | Count rem 5 == 0 -> 127 | async_start_set_net_ticktime_daemons( 128 | Time, da_nodes(nodes(connected))), 129 | ok; 130 | true -> 131 | ok 132 | end, 133 | set_net_ticktime_daemon_loop(Time, Count + 1) 134 | end. 135 | 136 | da_nodes(Nodes) -> 137 | lists:sort([node()|Nodes]). % Always include myself 138 | 139 | set_net_ticktime(Time) -> 140 | case net_kernel:set_net_ticktime(Time) of 141 | {Status, _} -> 142 | Status; 143 | A when is_atom(A) -> 144 | A 145 | end. 146 | 147 | -spec net_ticktime_active() -> boolean(). 148 | net_ticktime_active() -> 149 | case catch riak_core_capability:get({riak_core, net_ticktime}) of 150 | true -> 151 | true; 152 | _ -> 153 | false 154 | end. 155 | -------------------------------------------------------------------------------- /src/riak_core_new_claim.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riak_core: Core Riak Application 4 | %% 5 | %% Copyright (c) 2007-2011 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | %% 23 | %% @doc This module is a pass-thru to `riak_core_claim' for backwards 24 | %% compatability. 25 | 26 | -module(riak_core_new_claim). 27 | -export([new_wants_claim/2, new_choose_claim/2]). 28 | 29 | %% @deprecated 30 | %% 31 | %% @doc This exists for the sole purpose of backwards compatability. 32 | new_wants_claim(Ring, Node) -> 33 | riak_core_claim:wants_claim_v2(Ring, Node). 34 | 35 | %% @deprecated 36 | %% 37 | %% @doc This exists for the sole purpose of backwards compatability. 38 | new_choose_claim(Ring, Node) -> 39 | riak_core_claim:choose_claim_v2(Ring, Node). 40 | -------------------------------------------------------------------------------- /src/riak_core_node_watcher_events.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riak_core: Core Riak Application 4 | %% 5 | %% Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | -module(riak_core_node_watcher_events). 23 | 24 | -behaviour(gen_event). 25 | 26 | %% API 27 | -export([start_link/0, 28 | add_handler/2, 29 | add_sup_handler/2, 30 | add_guarded_handler/2, 31 | add_callback/1, 32 | add_sup_callback/1, 33 | add_guarded_callback/1, 34 | service_update/1]). 35 | 36 | %% gen_event callbacks 37 | -export([init/1, handle_event/2, handle_call/2, 38 | handle_info/2, terminate/2, code_change/3]). 39 | 40 | -record(state, { callback }). 41 | 42 | %% =================================================================== 43 | %% API functions 44 | %% =================================================================== 45 | 46 | start_link() -> 47 | gen_event:start_link({local, ?MODULE}). 48 | 49 | add_handler(Handler, Args) -> 50 | gen_event:add_handler(?MODULE, Handler, Args). 51 | 52 | add_sup_handler(Handler, Args) -> 53 | gen_event:add_sup_handler(?MODULE, Handler, Args). 54 | 55 | add_guarded_handler(Handler, Args) -> 56 | riak_core:add_guarded_event_handler(?MODULE, Handler, Args). 57 | 58 | add_callback(Fn) when is_function(Fn) -> 59 | gen_event:add_handler(?MODULE, {?MODULE, make_ref()}, [Fn]). 60 | 61 | add_sup_callback(Fn) when is_function(Fn) -> 62 | gen_event:add_sup_handler(?MODULE, {?MODULE, make_ref()}, [Fn]). 63 | 64 | add_guarded_callback(Fn) when is_function(Fn) -> 65 | riak_core:add_guarded_event_handler(?MODULE, {?MODULE, make_ref()}, [Fn]). 66 | 67 | service_update(Services) -> 68 | gen_event:notify(?MODULE, {service_update, Services}). 69 | 70 | 71 | %% =================================================================== 72 | %% gen_event callbacks 73 | %% =================================================================== 74 | 75 | init([Fn]) -> 76 | %% Get the initial list of available services 77 | Fn(riak_core_node_watcher:services()), 78 | {ok, #state { callback = Fn }}. 79 | 80 | handle_event({service_update, Services}, State) -> 81 | (State#state.callback)(Services), 82 | {ok, State}. 83 | 84 | handle_call(_Request, State) -> 85 | {ok, ok, State}. 86 | 87 | handle_info(_Info, State) -> 88 | {ok, State}. 89 | 90 | terminate(_Reason, _State) -> 91 | ok. 92 | 93 | code_change(_OldVsn, State, _Extra) -> 94 | {ok, State}. 95 | 96 | -------------------------------------------------------------------------------- /src/riak_core_nodeid.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riak_core: Core Node Id 4 | %% 5 | %% Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | %% Return a binary to use as an identifier for this node. 23 | %% Initially this is just hash the node name, so there is a small 24 | %% chance of collisions. 25 | %% ------------------------------------------------------------------- 26 | -module(riak_core_nodeid). 27 | -export([get/0]). 28 | 29 | get() -> 30 | Id = erlang:crc32(term_to_binary(node())), 31 | <>. 32 | -------------------------------------------------------------------------------- /src/riak_core_pw_auth.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2013 Basho Technologies, Inc. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | -module(riak_core_pw_auth). 22 | 23 | -export([hash_password/1, check_password/5]). 24 | 25 | %% TOOD should make these configurable in app.config 26 | -define(SALT_LENGTH, 16). 27 | -define(HASH_ITERATIONS, 65536). 28 | %% TODO this should call a default_hash_func() function to get default based on erlang version 29 | -define(HASH_FUNCTION, sha). 30 | -define(AUTH_NAME, pbkdf2). 31 | 32 | %% @doc Hash a plaintext password, returning hashed password and algorithm details 33 | hash_password(BinaryPass) when is_binary(BinaryPass) -> 34 | % TODO: Do something more with the salt? 35 | % Generate salt the simple way 36 | Salt = riak_core_rand:rand_bytes(?SALT_LENGTH), 37 | 38 | % Hash the original password and store as hex 39 | {ok, HashedPass} = pbkdf2:pbkdf2(?HASH_FUNCTION, BinaryPass, Salt, ?HASH_ITERATIONS), 40 | HexPass = pbkdf2:to_hex(HashedPass), 41 | {ok, HexPass, ?AUTH_NAME, ?HASH_FUNCTION, Salt, ?HASH_ITERATIONS}. 42 | 43 | 44 | %% @doc Check a plaintext password with a hashed password 45 | check_password(BinaryPass, HashedPassword, HashFunction, Salt, HashIterations) when is_binary(BinaryPass) -> 46 | 47 | % Hash EnteredPassword to compare to HashedPassword 48 | {ok, HashedPass} = pbkdf2:pbkdf2(HashFunction, BinaryPass, Salt, HashIterations), 49 | HexPass = pbkdf2:to_hex(HashedPass), 50 | pbkdf2:compare_secure(HexPass, HashedPassword). 51 | -------------------------------------------------------------------------------- /src/riak_core_rand.erl: -------------------------------------------------------------------------------- 1 | %% Generalized random module that offers a backwards compatible API 2 | %% around some of the changes in rand, crypto and for time units. 3 | 4 | -module(riak_core_rand). 5 | 6 | %% API 7 | -export([ 8 | uniform/0, 9 | uniform/1, 10 | uniform_s/2, 11 | seed/0, 12 | seed/1, 13 | rand_seed/0, 14 | rand_bytes/1 15 | ]). 16 | 17 | %% As the algorithm is not changed in any place we can use the default 18 | %% algorithm for all call here. 19 | -define(ALGO, exsplus). 20 | 21 | %%%=================================================================== 22 | %%% New (r19+) rand style functions 23 | %%%=================================================================== 24 | -ifdef(rand). 25 | uniform() -> 26 | rand:uniform(). 27 | 28 | uniform(N) -> 29 | rand:uniform(N). 30 | 31 | %% The old random:uniform_s took a 3 touple however this is no longer 32 | %% the case, so what we need to do if we see such a situation is to first 33 | %% create a state using seed_s (which can take the old data) and then 34 | %% using uniform_s with this newly generated state. 35 | %% 36 | %% Note that seed_s does **not** change the current seed but just 37 | %% create a new seed state. 38 | uniform_s(N, {A, B, C}) -> 39 | State = rand:seed_s(?ALGO, {A, B, C}), 40 | rand:uniform_s(N, State); 41 | uniform_s(N, State) -> 42 | rand:uniform_s(N, State). 43 | 44 | seed() -> 45 | rand:seed(?ALGO). 46 | 47 | %% We are a bit tricky here, while random:seed did return the **prior** seed 48 | %% rand:seed will return the **new** seed. We can work around this by first 49 | %% getting the exported seed then using this instead. 50 | -spec seed({integer(),integer(),integer()} | rand:export_state()) -> 51 | rand:export_state() | undefined. 52 | seed({_, _, _} = Seed) -> 53 | Old = rand:export_seed(), 54 | _New = rand:seed(?ALGO, Seed), 55 | Old; 56 | seed(Seed) -> 57 | Old = rand:export_seed(), 58 | _New = rand:seed(Seed), 59 | Old. 60 | 61 | rand_bytes(Size) -> 62 | crypto:strong_rand_bytes(Size). 63 | 64 | -else. 65 | %%%=================================================================== 66 | %%% Old (r18) random style functions 67 | %%%=================================================================== 68 | 69 | uniform() -> 70 | random:uniform(). 71 | 72 | uniform(N) -> 73 | random:uniform(N). 74 | 75 | uniform_s(N, State) -> 76 | random:uniform_s(N, State). 77 | 78 | seed() -> 79 | random:seed(). 80 | 81 | seed({A, B, C}) -> 82 | random:seed({A, B, C}). 83 | 84 | rand_bytes(Size) -> 85 | crypto:strong_rand(Size). 86 | 87 | -endif. 88 | 89 | %%%=================================================================== 90 | %%% General functions 91 | %%%=================================================================== 92 | 93 | rand_seed() -> 94 | erlang:timestamp(). 95 | -------------------------------------------------------------------------------- /src/riak_core_repair.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2012 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | -module(riak_core_repair). 22 | -export([gen_filter/5, 23 | gen_range/3, 24 | gen_range_fun/2, 25 | gen_range_map/3]). 26 | 27 | -include("riak_core_vnode.hrl"). 28 | 29 | %% GTE = Greater Than or Equal 30 | %% LTE = Less Than or Equal 31 | -type range_wrap() :: {wrap, GTE::binary(), LTE::binary()}. 32 | -type range_nowrap() :: {nowrap, GTE::binary(), LTE::binary()}. 33 | -type hash_range() :: range_wrap() | range_nowrap(). 34 | 35 | %% =================================================================== 36 | %% Public API 37 | %% =================================================================== 38 | 39 | %% @doc Generate a `Filter' fun to use during partition repair. 40 | %% 41 | %% `Target' - Partition under repair. 42 | %% 43 | %% `Ring' - The ring to use for repair. 44 | %% 45 | %% `NValMap' - A map from bucket to `n_val', only custom buckets 46 | %% have entries, everything else uses default. 47 | %% 48 | %% `DefaultN' - The default `n_val'. 49 | %% 50 | %% `InfoFun' - A function which returns information about the key 51 | %% used to determine if it should be repaired or not. 52 | gen_filter(Target, Ring, NValMap, DefaultN, InfoFun) -> 53 | RangeMap = riak_core_repair:gen_range_map(Target, Ring, NValMap), 54 | Default = riak_core_repair:gen_range(Target, Ring, DefaultN), 55 | RangeFun = riak_core_repair:gen_range_fun(RangeMap, Default), 56 | fun(BKey) -> 57 | {Bucket, Hash} = InfoFun(BKey), 58 | case RangeFun(Bucket) of 59 | {nowrap, GTE, LTE} -> 60 | Hash >= GTE andalso Hash =< LTE; 61 | {wrap, GTE, LTE} -> 62 | Hash >= GTE orelse Hash =< LTE 63 | end 64 | end. 65 | 66 | %% @doc Generate the hash `Range' for a given `Target' partition and 67 | %% `NVal'. 68 | %% 69 | %% Note: The type of NVal should be pos_integer() but dialyzer says 70 | %% success typing is integer() and I don't have time for games. 71 | -spec gen_range(partition(), riak_core_ring:riak_core_ring(), integer()) -> 72 | hash_range(). 73 | gen_range(Target, Ring, NVal) -> 74 | CH = riak_core_ring:chash(Ring), 75 | Predecessors = chash:predecessors(<>, CH, NVal+1), 76 | [{FirstIdx, Node}|Rest] = Predecessors, 77 | Predecessors2 = [{FirstIdx-1, Node}|Rest], 78 | Predecessors3 = [<> || {I,_} <- Predecessors2], 79 | {A,B} = lists:splitwith(fun(E) -> E > <<0:160/integer>> end, Predecessors3), 80 | case B of 81 | [] -> 82 | %% In this case there is no "wrap" around the end of the 83 | %% ring so the range check it simply an inclusive 84 | %% inbetween. 85 | make_nowrap(A); 86 | 87 | [<<0:160/integer>>] -> 88 | %% In this case the 0th partition is first so no wrap 89 | %% actually occurred. 90 | make_nowrap(A ++ B); 91 | 92 | _ -> 93 | %% In this case there is a "wrap" around the end of the 94 | %% ring. Either the key is greater than or equal the 95 | %% largest or smaller than or equal to the smallest. 96 | make_wrap(A, B) 97 | end. 98 | 99 | 100 | %% @doc Generate the function that will return the hash range for a 101 | %% given `Bucket'. 102 | -spec gen_range_fun(list(), hash_range()) -> function(). 103 | gen_range_fun(RangeMap, Default) -> 104 | fun(Bucket) -> 105 | case lists:keyfind(Bucket, 1, RangeMap) of 106 | false -> Default; 107 | {_, Val} -> Val 108 | end 109 | end. 110 | 111 | %% @doc Generate the map from bucket `B' to hash `Range' that a key 112 | %% must fall into to be included for repair on the `Target' 113 | %% partition. 114 | -spec gen_range_map(partition(), riak_core_ring:riak_core_ring(), list()) -> 115 | [{Bucket::binary(), Range::hash_range()}]. 116 | gen_range_map(Target, Ring, NValMap) -> 117 | [{I, gen_range(Target, Ring, N)} || {I, N} <- NValMap, N > 1]. 118 | 119 | 120 | %% =================================================================== 121 | %% Internal 122 | %% =================================================================== 123 | 124 | %% @private 125 | %% 126 | %% @doc Make the `nowrap' tuple representing the range a key hash must 127 | %% fall into. 128 | -spec make_nowrap(list()) -> range_nowrap(). 129 | make_nowrap(RingSlice) -> 130 | Slice2 = lists:sort(RingSlice), 131 | GTE = hd(Slice2), 132 | LTE = lists:last(Slice2), 133 | {nowrap, GTE, LTE}. 134 | 135 | %% @private 136 | %% 137 | %% @doc Make `wrap' tuple representing the two ranges, relative to 0, 138 | %% that a key hash must fall into. 139 | %% 140 | %% `RingSliceAfter' contains entries after the wrap around the 0th 141 | %% partition. `RingSliceBefore' are entries before wrap and the 142 | %% including the wrap at partition 0. 143 | -spec make_wrap(list(), list()) -> range_wrap(). 144 | make_wrap(RingSliceAfter, RingSliceBefore) -> 145 | LTE = hd(RingSliceAfter), 146 | %% know first element of RingSliceBefore is 0 147 | Before2 = tl(RingSliceBefore), 148 | GTE = lists:last(Before2), 149 | {wrap, GTE, LTE}. 150 | 151 | -------------------------------------------------------------------------------- /src/riak_core_ring_events.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riak_core: Core Riak Application 4 | %% 5 | %% Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | -module(riak_core_ring_events). 23 | 24 | -behaviour(gen_event). 25 | 26 | %% API 27 | -export([start_link/0, 28 | add_handler/2, 29 | add_sup_handler/2, 30 | add_guarded_handler/2, 31 | add_callback/1, 32 | add_sup_callback/1, 33 | add_guarded_callback/1, 34 | ring_update/1, 35 | force_update/0, 36 | ring_sync_update/1, 37 | force_sync_update/0]). 38 | 39 | %% gen_event callbacks 40 | -export([init/1, handle_event/2, handle_call/2, 41 | handle_info/2, terminate/2, code_change/3]). 42 | 43 | -record(state, { callback }). 44 | 45 | %% =================================================================== 46 | %% API functions 47 | %% =================================================================== 48 | 49 | start_link() -> 50 | gen_event:start_link({local, ?MODULE}). 51 | 52 | add_handler(Handler, Args) -> 53 | gen_event:add_handler(?MODULE, Handler, Args). 54 | 55 | add_sup_handler(Handler, Args) -> 56 | gen_event:add_sup_handler(?MODULE, Handler, Args). 57 | 58 | add_guarded_handler(Handler, Args) -> 59 | riak_core:add_guarded_event_handler(?MODULE, Handler, Args). 60 | 61 | add_callback(Fn) when is_function(Fn) -> 62 | gen_event:add_handler(?MODULE, {?MODULE, make_ref()}, [Fn]). 63 | 64 | add_sup_callback(Fn) when is_function(Fn) -> 65 | gen_event:add_sup_handler(?MODULE, {?MODULE, make_ref()}, [Fn]). 66 | 67 | add_guarded_callback(Fn) when is_function(Fn) -> 68 | riak_core:add_guarded_event_handler(?MODULE, {?MODULE, make_ref()}, [Fn]). 69 | 70 | force_update() -> 71 | {ok, Ring} = riak_core_ring_manager:get_my_ring(), 72 | ring_update(Ring). 73 | 74 | ring_update(Ring) -> 75 | gen_event:notify(?MODULE, {ring_update, Ring}). 76 | 77 | force_sync_update() -> 78 | {ok, Ring} = riak_core_ring_manager:get_my_ring(), 79 | ring_sync_update(Ring). 80 | 81 | ring_sync_update(Ring) -> 82 | gen_event:sync_notify(?MODULE, {ring_update, Ring}). 83 | 84 | %% =================================================================== 85 | %% gen_event callbacks 86 | %% =================================================================== 87 | 88 | init([Fn]) -> 89 | {ok, Ring} = riak_core_ring_manager:get_my_ring(), 90 | Fn(Ring), 91 | {ok, #state { callback = Fn }}. 92 | 93 | handle_event({ring_update, Ring}, State) -> 94 | (State#state.callback)(Ring), 95 | {ok, State}. 96 | 97 | handle_call(_Request, State) -> 98 | {ok, ok, State}. 99 | 100 | handle_info(_Info, State) -> 101 | {ok, State}. 102 | 103 | terminate(_Reason, _State) -> 104 | ok. 105 | 106 | code_change(_OldVsn, State, _Extra) -> 107 | {ok, State}. 108 | 109 | -------------------------------------------------------------------------------- /src/riak_core_send_msg.erl: -------------------------------------------------------------------------------- 1 | %% 2 | %% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. 3 | %% 4 | %% This file is provided to you under the Apache License, 5 | %% Version 2.0 (the "License"); you may not use this file 6 | %% except in compliance with the License. You may obtain 7 | %% a copy of the License at 8 | %% 9 | %% http://www.apache.org/licenses/LICENSE-2.0 10 | %% 11 | %% Unless required by applicable law or agreed to in writing, 12 | %% software distributed under the License is distributed on an 13 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | %% KIND, either express or implied. See the License for the 15 | %% specific language governing permissions and limitations 16 | %% under the License. 17 | %% 18 | %% ------------------------------------------------------------------- 19 | 20 | %% @doc OTP equivalents for sending reply- and event-like things 21 | %% without blocking the sender. 22 | 23 | -module(riak_core_send_msg). 24 | 25 | -export([reply_unreliable/2, 26 | cast_unreliable/2, 27 | send_event_unreliable/2, 28 | bang_unreliable/2]). 29 | 30 | -ifdef(TEST). 31 | -ifdef(PULSE). 32 | -compile(export_all). 33 | -compile({parse_transform, pulse_instrument}). 34 | -compile({pulse_replace_module, [{gen_fsm, pulse_gen_fsm}, 35 | {gen_fsm_compat, pulse_gen_fsm}, 36 | {gen_server, pulse_gen_server}]}). 37 | -endif. 38 | -endif. 39 | 40 | %% NOTE: We'ed peeked inside gen_server.erl's guts to see its internals. 41 | reply_unreliable({To, Tag}, Reply) -> 42 | bang_unreliable(To, {Tag, Reply}). 43 | 44 | cast_unreliable(Dest, Request) -> 45 | bang_unreliable(Dest, {'$gen_cast', Request}). 46 | 47 | %% NOTE: We'ed peeked inside gen_fsm.erl's guts to see its internals. 48 | send_event_unreliable({global, _Name} = GlobalTo, Event) -> 49 | erlang:error({unimplemented_send, GlobalTo, Event}); 50 | send_event_unreliable({via, _Mod, _Name} = ViaTo, Event) -> 51 | erlang:error({unimplemented_send, ViaTo, Event}); 52 | send_event_unreliable(Name, Event) -> 53 | bang_unreliable(Name, {'$gen_event', Event}), 54 | ok. 55 | 56 | bang_unreliable(Dest, Msg) -> 57 | catch erlang:send(Dest, Msg, [noconnect, nosuspend]), 58 | Msg. 59 | -------------------------------------------------------------------------------- /src/riak_core_stat_cache.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | -module(riak_core_stat_cache). 21 | 22 | -behaviour(gen_server). 23 | 24 | -export([start_link/0, 25 | stop/0]). 26 | 27 | -export([register_app/2, 28 | get_stats/1]). 29 | 30 | -export([init/1, 31 | handle_call/3, 32 | handle_cast/2, 33 | handle_info/2, 34 | terminate/2, 35 | code_change/3]). 36 | 37 | -record(st, {apps = []}). 38 | 39 | register_app(App, ProduceMFA) -> 40 | gen_server:call(?MODULE, {register, App, ProduceMFA}), 41 | ok. 42 | 43 | %% This function should not be called by code that's been adapted to 44 | %% exometer. Thus, it must return the old format, and assume that the 45 | %% 46 | get_stats(App) -> 47 | T1 = os:timestamp(), 48 | Result = case get_app(App) of 49 | {_, {M, F, A}} -> 50 | apply(M, F, A); 51 | false -> 52 | [] 53 | end, 54 | T2 = os:timestamp(), 55 | {ok, Result, timer:now_diff(T2, T1)}. 56 | 57 | %% The gen_server mainly exists as a stub, since legacy test suites expect 58 | %% riak_core_stat_cache:start_link() to return {ok, Pid}. It also keeps track 59 | %% of the produce_stats() callbacks for each registered app. 60 | 61 | start_link() -> 62 | gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). 63 | 64 | stop() -> 65 | ok. 66 | 67 | init(_) -> {ok, #st{}}. 68 | 69 | handle_call({register, App, MFA}, _, #st{apps = Apps} = S) -> 70 | {reply, ok, S#st{apps = lists:keystore(App, 1, Apps, {App, MFA})}}; 71 | handle_call({get_app, App}, _, #st{apps = Apps} = S) -> 72 | {reply, lists:keyfind(App, 1, Apps), S}; 73 | handle_call(_, _, S) -> 74 | {reply, {error,not_supported}, S}. 75 | 76 | handle_cast(_, S) -> {noreply, S}. 77 | handle_info(_, S) -> {noreply, S}. 78 | terminate(_, _) -> ok. 79 | code_change(_, S, _) -> {ok, S}. 80 | 81 | %% -- end gen_server 82 | 83 | get_app(App) -> 84 | gen_server:call(?MODULE, {get_app, App}). 85 | -------------------------------------------------------------------------------- /src/riak_core_stat_calc_sup.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% Copyright (c) 2007-2011 Basho Technologies, Inc. All Rights Reserved. 3 | %% 4 | %% This file is provided to you under the Apache License, 5 | %% Version 2.0 (the "License"); you may not use this file 6 | %% except in compliance with the License. You may obtain 7 | %% a copy of the License at 8 | %% 9 | %% http://www.apache.org/licenses/LICENSE-2.0 10 | %% 11 | %% Unless required by applicable law or agreed to in writing, 12 | %% software distributed under the License is distributed on an 13 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | %% KIND, either express or implied. See the License for the 15 | %% specific language governing permissions and limitations 16 | %% under the License. 17 | %% 18 | %% ------------------------------------------------------------------- 19 | -module(riak_core_stat_calc_sup). 20 | -behaviour(supervisor). 21 | -export([start_link/0, init/1]). 22 | -export([calc_proc/1, stop_proc/1]). 23 | 24 | %% Helper macro for declaring children of supervisor 25 | -define(CHILD(I, Type, Timeout), {I, {riak_core_stat_calc_proc, start_link, [I]}, permanent, Timeout, Type, [riak_core_stat_calc_proc]}). 26 | -define(CHILD(I, Type), ?CHILD(I, Type, 5000)). 27 | 28 | start_link() -> 29 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 30 | 31 | init([]) -> 32 | {ok, {{one_for_one, 5, 10}, []}}. 33 | 34 | %% start a process for the given stat 35 | %% {@see riak_core_stat_calc_proc} 36 | calc_proc(Stat) -> 37 | Ref = calc_proc_ref(Stat), 38 | Pid = case supervisor:start_child(?MODULE, Ref) of 39 | {ok, Child} -> Child; 40 | {error, {already_started, Child}} -> Child 41 | end, 42 | Pid. 43 | 44 | stop_proc(Stat) -> 45 | _ = supervisor:terminate_child(?MODULE, Stat), 46 | _ = supervisor:delete_child(?MODULE, Stat), 47 | ok. 48 | 49 | %% @private 50 | calc_proc_ref(Stat) -> 51 | ?CHILD(Stat, worker). 52 | -------------------------------------------------------------------------------- /src/riak_core_stat_q.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2007-2012 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | %% @doc riak_core_stat_q is an interface to query folsom stats 22 | %% To use, call `get_stats/1' with a query `Path'. 23 | %% A `Path' is a list of atoms | binaries. The module creates a set 24 | %% of `ets:select/1' guards, one for each element in `Path' 25 | %% For each stat that has a key that matches `Path' we calculate the 26 | %% current value and return it. 27 | 28 | -module(riak_core_stat_q). 29 | 30 | -export([get_stats/1, calculate_stats/1, get_stat/1, calc_stat/1, 31 | stat_return/1, log_error/3, calc_gauge/1]). 32 | 33 | -export_type([path/0, 34 | stat_name/0]). 35 | 36 | -type path() :: [] | [atom()|binary()]. 37 | -type stats() :: [stat()]. 38 | -type stat() :: {stat_name(), stat_value()}. 39 | -type stat_name() :: list(). 40 | -type stat_value() :: integer() | [tuple()]. 41 | 42 | %% @doc To allow for namespacing, and adding richer dimensions, stats 43 | %% are named with a tuple key. The key (like `{riak_kv, node, gets}' or 44 | %% `{riak_kv, vnode, puts, time}') can 45 | %% be seen as an hierarchical path. With `riak_kv' at the root and 46 | %% the other elements as branches / leaves. 47 | %% This module allows us to get only the stats at and below a particular key. 48 | %% `Path' is a list of atoms or the empty list. 49 | %% an example path might be `[riak_kv]' which will return every 50 | %% stat that has `riak_kv' in the first element of its key tuple. 51 | %% You may use the atom '_' at any point 52 | %% in `Path' as a wild card. 53 | -spec get_stats(path()) -> stats(). 54 | get_stats(Path) -> 55 | exometer:get_values(Path). 56 | %% %% get all the stats that are at Path 57 | %% calculate_stats(exometer:select( 58 | %% [{ {Path ++ '_','_',enabled}, [], ['$_'] }])). 59 | 60 | calculate_stats(NamesAndTypes) -> 61 | [{Name, get_stat(Name)} || {Name, _, _} <- NamesAndTypes]. 62 | 63 | %% Create/lookup a cache/calculation process 64 | get_stat(Stat) -> 65 | exometer:get_value(Stat). 66 | 67 | %% Encapsulate getting a stat value from exometer. 68 | %% 69 | %% If for any reason we can't get a stats value 70 | %% return 'unavailable'. 71 | %% @TODO experience shows that once a stat is 72 | %% broken it stays that way. Should we delete 73 | %% stats that are broken? 74 | calc_stat({Name, _Type}) when is_tuple(Name) -> 75 | stat_return(exometer:get_value([riak_core_stat:prefix()|tuple_to_list(Name)])); 76 | calc_stat({[_|_] = Name, _Type}) -> 77 | stat_return(exometer:get_value([riak_core_stat:prefix()|Name])). 78 | 79 | stat_return({error,not_found}) -> unavailable; 80 | stat_return({ok, Value}) -> Value. 81 | 82 | log_error(StatName, ErrClass, ErrReason) -> 83 | lager:warning("Failed to calculate stat ~p with ~p:~p", [StatName, ErrClass, ErrReason]). 84 | 85 | %% some crazy people put funs in gauges (exometer has a 'function' metric) 86 | %% so that they can have a consistent interface 87 | %% to access stats from disperate sources 88 | calc_gauge({function, Mod, Fun}) -> 89 | Mod:Fun(); 90 | calc_gauge(Val) -> 91 | Val. 92 | -------------------------------------------------------------------------------- /src/riak_core_stat_sup.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riak_core: Core Riak Application 4 | %% 5 | %% Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | -module(riak_core_stat_sup). 24 | 25 | -behaviour(supervisor). 26 | 27 | %% API 28 | -export([start_link/0]). 29 | 30 | %% Supervisor callbacks 31 | -export([init/1]). 32 | 33 | %% Helper macro for declaring children of supervisor 34 | -define(CHILD(I, Type, Timeout), {I, {I, start_link, []}, permanent, Timeout, Type, [I]}). 35 | -define(CHILD(I, Type), ?CHILD(I, Type, 5000)). 36 | 37 | %% =================================================================== 38 | %% API functions 39 | %% =================================================================== 40 | 41 | start_link() -> 42 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 43 | 44 | %% =================================================================== 45 | %% Supervisor callbacks 46 | %% =================================================================== 47 | 48 | init([]) -> 49 | Children = lists:flatten( 50 | [ 51 | ?CHILD(riak_core_stats_sup, supervisor) 52 | ]), 53 | 54 | {ok, {{rest_for_one, 10, 10}, Children}}. 55 | -------------------------------------------------------------------------------- /src/riak_core_stat_xform.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | -module(riak_core_stat_xform). 21 | 22 | -export([parse_transform/2]). 23 | 24 | 25 | parse_transform(Forms, _) -> 26 | case os:getenv("RIAK_CORE_STAT_PREFIX") of 27 | false -> 28 | Forms; 29 | Str -> 30 | transform(Forms, list_to_atom(Str)) 31 | end. 32 | 33 | transform([{function,L,prefix,0,[_]}|T], Pfx) -> 34 | [{function, L, prefix, 0, 35 | [{clause, L, [], [], [{atom,L,Pfx}]}]}|T]; 36 | transform([H|T], Pfx) -> 37 | [H | transform(T, Pfx)]; 38 | transform([], _) -> 39 | []. 40 | 41 | 42 | -------------------------------------------------------------------------------- /src/riak_core_stats_sup.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% Copyright (c) 2007-2011 Basho Technologies, Inc. All Rights Reserved. 3 | %% 4 | %% This file is provided to you under the Apache License, 5 | %% Version 2.0 (the "License"); you may not use this file 6 | %% except in compliance with the License. You may obtain 7 | %% a copy of the License at 8 | %% 9 | %% http://www.apache.org/licenses/LICENSE-2.0 10 | %% 11 | %% Unless required by applicable law or agreed to in writing, 12 | %% software distributed under the License is distributed on an 13 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | %% KIND, either express or implied. See the License for the 15 | %% specific language governing permissions and limitations 16 | %% under the License. 17 | %% 18 | %% ------------------------------------------------------------------- 19 | -module(riak_core_stats_sup). 20 | -behaviour(supervisor). 21 | -export([start_link/0, init/1]). 22 | -export([start_server/1, stop_server/1]). 23 | 24 | %% Helper macro for declaring children of supervisor 25 | -define(CHILD(I, Type, Timeout), {I, {I, start_link, []}, permanent, Timeout, Type, [I]}). 26 | -define(CHILD(I, Type), ?CHILD(I, Type, 5000)). 27 | 28 | start_link() -> 29 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 30 | 31 | init([]) -> 32 | Children = [stat_server(Mod) || {_App, Mod} <- riak_core:stat_mods()], 33 | {ok, {{one_for_one, 5, 10}, [?CHILD(riak_core_stat_cache, worker)|Children]}}. 34 | 35 | start_server(Mod) -> 36 | Ref = stat_server(Mod), 37 | Pid = case supervisor:start_child(?MODULE, Ref) of 38 | {ok, Child} -> Child; 39 | {error, {already_started, Child}} -> Child 40 | end, 41 | Pid. 42 | 43 | stop_server(Mod) -> 44 | _ = supervisor:terminate_child(?MODULE, Mod), 45 | _ = supervisor:delete_child(?MODULE, Mod), 46 | ok. 47 | 48 | %% @private 49 | stat_server(Mod) -> 50 | ?CHILD(Mod, worker). 51 | -------------------------------------------------------------------------------- /src/riak_core_sup.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riak_core: Core Riak Application 4 | %% 5 | %% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | -module(riak_core_sup). 24 | 25 | -behaviour(supervisor). 26 | 27 | -include("riak_core_bg_manager.hrl"). 28 | 29 | %% API 30 | -export([start_link/0]). 31 | -export([ensembles_enabled/0]). 32 | 33 | %% Supervisor callbacks 34 | -export([init/1]). 35 | 36 | %% Helper macro for declaring children of supervisor 37 | -define(CHILD(I, Type, Timeout, Args), {I, {I, start_link, Args}, permanent, Timeout, Type, [I]}). 38 | -define(CHILD(I, Type, Timeout), ?CHILD(I, Type, Timeout, [])). 39 | -define(CHILD(I, Type), ?CHILD(I, Type, 5000)). 40 | 41 | %% =================================================================== 42 | %% API functions 43 | %% =================================================================== 44 | 45 | start_link() -> 46 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 47 | 48 | %% =================================================================== 49 | %% Supervisor callbacks 50 | %% =================================================================== 51 | 52 | init([]) -> 53 | DistMonEnabled = app_helper:get_env(riak_core, enable_dist_mon, 54 | true), 55 | {ok, Root} = application:get_env(riak_core, platform_data_dir), 56 | 57 | EnsembleSup = {riak_ensemble_sup, 58 | {riak_ensemble_sup, start_link, [Root]}, 59 | permanent, 30000, supervisor, [riak_ensemble_sup]}, 60 | 61 | Children = lists:flatten( 62 | [?CHILD(riak_core_bg_manager, worker), 63 | ?CHILD(riak_core_sysmon_minder, worker), 64 | ?CHILD(riak_core_vnode_sup, supervisor, 305000), 65 | ?CHILD(riak_core_eventhandler_sup, supervisor), 66 | [?CHILD(riak_core_dist_mon, worker) || DistMonEnabled], 67 | ?CHILD(riak_core_handoff_sup, supervisor), 68 | ?CHILD(riak_core_ring_events, worker), 69 | ?CHILD(riak_core_ring_manager, worker), 70 | ?CHILD(riak_core_metadata_evt_sup, supervisor), 71 | ?CHILD(riak_core_metadata_manager, worker), 72 | ?CHILD(riak_core_metadata_hashtree, worker), 73 | ?CHILD(riak_core_broadcast, worker), 74 | ?CHILD(riak_core_vnode_proxy_sup, supervisor), 75 | ?CHILD(riak_core_node_watcher_events, worker), 76 | ?CHILD(riak_core_node_watcher, worker), 77 | ?CHILD(riak_core_vnode_manager, worker), 78 | ?CHILD(riak_core_capability, worker), 79 | ?CHILD(riak_core_gossip, worker), 80 | ?CHILD(riak_core_claimant, worker), 81 | ?CHILD(riak_core_table_owner, worker), 82 | ?CHILD(riak_core_stat_sup, supervisor), 83 | [EnsembleSup || ensembles_enabled()] 84 | ]), 85 | 86 | {ok, {{one_for_one, 10, 10}, Children}}. 87 | 88 | ensembles_enabled() -> 89 | Exists = (code:which(riak_ensemble_sup) =/= non_existing), 90 | Enabled = app_helper:get_env(riak_core, enable_consensus, false), 91 | Exists and Enabled. 92 | -------------------------------------------------------------------------------- /src/riak_core_sysmon_minder.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riak_core: Core Riak Application 4 | %% 5 | %% Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | -module(riak_core_sysmon_minder). 24 | 25 | -behaviour(gen_server). 26 | 27 | %% API 28 | -export([start_link/0]). 29 | 30 | %% gen_server callbacks 31 | -export([init/1, handle_call/3, handle_cast/2, handle_info/2, 32 | terminate/2, code_change/3]). 33 | 34 | -record(state, {}). 35 | 36 | %%%=================================================================== 37 | %%% API 38 | %%%=================================================================== 39 | 40 | %%-------------------------------------------------------------------- 41 | %% @doc 42 | %% Starts the server 43 | %% 44 | %% @spec start_link() -> {ok, Pid} | ignore | {error, Error} 45 | %% @end 46 | %%-------------------------------------------------------------------- 47 | start_link() -> 48 | gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). 49 | 50 | %%%=================================================================== 51 | %%% gen_server callbacks 52 | %%%=================================================================== 53 | 54 | %%-------------------------------------------------------------------- 55 | %% @private 56 | %% @doc 57 | %% Initializes the server 58 | %% 59 | %% @spec init(Args) -> {ok, State} | 60 | %% {ok, State, Timeout} | 61 | %% ignore | 62 | %% {stop, Reason} 63 | %% @end 64 | %%-------------------------------------------------------------------- 65 | init([]) -> 66 | %% Add our system_monitor event handler. We do that here because 67 | %% we have a process at our disposal (i.e. ourself) to receive the 68 | %% notification in the very unlikely event that the 69 | %% riak_core_sysmon_handler has crashed and been removed from the 70 | %% riak_sysmon_handler gen_event server. (If we had a supervisor 71 | %% or app-starting process add the handler, then if the handler 72 | %% crashes, nobody will act on the crash notification.) 73 | riak_core_sysmon_handler:add_handler(), 74 | 75 | {ok, #state{}}. 76 | 77 | %%-------------------------------------------------------------------- 78 | %% @private 79 | %% @doc 80 | %% Handling call messages 81 | %% 82 | %% @spec handle_call(Request, From, State) -> 83 | %% {reply, Reply, State} | 84 | %% {reply, Reply, State, Timeout} | 85 | %% {noreply, State} | 86 | %% {noreply, State, Timeout} | 87 | %% {stop, Reason, Reply, State} | 88 | %% {stop, Reason, State} 89 | %% @end 90 | %%-------------------------------------------------------------------- 91 | handle_call(_Request, _From, State) -> 92 | Reply = ok, 93 | {reply, Reply, State}. 94 | 95 | %%-------------------------------------------------------------------- 96 | %% @private 97 | %% @doc 98 | %% Handling cast messages 99 | %% 100 | %% @spec handle_cast(Msg, State) -> {noreply, State} | 101 | %% {noreply, State, Timeout} | 102 | %% {stop, Reason, State} 103 | %% @end 104 | %%-------------------------------------------------------------------- 105 | handle_cast(_Msg, State) -> 106 | {noreply, State}. 107 | 108 | %%-------------------------------------------------------------------- 109 | %% @private 110 | %% @doc 111 | %% Handling all non call/cast messages 112 | %% 113 | %% @spec handle_info(Info, State) -> {noreply, State} | 114 | %% {noreply, State, Timeout} | 115 | %% {stop, Reason, State} 116 | %% @end 117 | %%-------------------------------------------------------------------- 118 | handle_info({gen_event_EXIT, riak_core_sysmon_handler, _}, State) -> 119 | %% SASL will create an error message, no need for us to duplicate it. 120 | %% 121 | %% Our handler should never crash, but it did indeed crash. If 122 | %% there's a pathological condition somewhere that's generating 123 | %% lots of unforseen things that crash core's custom handler, we 124 | %% could make things worse by jumping back into the exploding 125 | %% volcano. Wait a little bit before jumping back. Besides, the 126 | %% system_monitor data is nice but is not critical: there is no 127 | %% need to make things worse if things are indeed bad, and if we 128 | %% miss a few seconds of system_monitor events, the world will not 129 | %% end. 130 | timer:sleep(2*1000), 131 | riak_core_sysmon_handler:add_handler(), 132 | {noreply, State}; 133 | handle_info(_Info, State) -> 134 | {noreply, State}. 135 | 136 | %%-------------------------------------------------------------------- 137 | %% @private 138 | %% @doc 139 | %% This function is called by a gen_server when it is about to 140 | %% terminate. It should be the opposite of Module:init/1 and do any 141 | %% necessary cleaning up. When it returns, the gen_server terminates 142 | %% with Reason. The return value is ignored. 143 | %% 144 | %% @spec terminate(Reason, State) -> void() 145 | %% @end 146 | %%-------------------------------------------------------------------- 147 | terminate(_Reason, _State) -> 148 | ok. 149 | 150 | %%-------------------------------------------------------------------- 151 | %% @private 152 | %% @doc 153 | %% Convert process state when code is changed 154 | %% 155 | %% @spec code_change(OldVsn, State, Extra) -> {ok, NewState} 156 | %% @end 157 | %%-------------------------------------------------------------------- 158 | code_change(_OldVsn, State, _Extra) -> 159 | {ok, State}. 160 | 161 | %%%=================================================================== 162 | %%% Internal functions 163 | %%%=================================================================== 164 | -------------------------------------------------------------------------------- /src/riak_core_table_owner.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% Copyright (c) 2016 Basho Technologies, Inc. All Rights Reserved. 3 | %% 4 | %% This file is provided to you under the Apache License, 5 | %% Version 2.0 (the "License"); you may not use this file 6 | %% except in compliance with the License. You may obtain 7 | %% a copy of the License at 8 | %% 9 | %% http://www.apache.org/licenses/LICENSE-2.0 10 | %% 11 | %% Unless required by applicable law or agreed to in writing, 12 | %% software distributed under the License is distributed on an 13 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | %% KIND, either express or implied. See the License for the 15 | %% specific language governing permissions and limitations 16 | %% under the License. 17 | %% 18 | %% ------------------------------------------------------------------- 19 | %% 20 | %% @doc

ETS tables, when created with the `ets:new/2' function, have a single 21 | %% owning process. If the owning process exits, then any ETS tables associated 22 | %% with that process are deleted. The purpose of the `riak_core_table_owner' 23 | %% module (which is a gen_server process) is to serve as the owning process for 24 | %% ETS tables that do not otherwise have an obvious owning process. For example, 25 | %% the `riak_core_throttle' module uses an ETS table for maintaining its state, 26 | %% but it is not itself a process and therefore the owning process for it ETS 27 | %% table is not clear. In this case, `riak_core_table_owner' can be used to 28 | %% create and own the ETS table on its behalf.

29 | %% 30 | %%

It is important that this process never crashes, as that would lead to 31 | %% loss of data. Therefore, a defensive approach is taken and any calls to 32 | %% external modules are protected with the try/catch mechanism.

33 | %% 34 | %%

Note that this first iteration does not provide any API functions for 35 | %% reading or writing data in ETS tables and therefore is appropriate only for 36 | %% named public ETS tables. In order to be more broadly useful, future 37 | %% enhancements to this module should include API functions for efficiently 38 | %% reading and writing ETS data, preferably without going through a gen_server 39 | %% call.

40 | -module(riak_core_table_owner). 41 | 42 | -behaviour(gen_server). 43 | 44 | %% API 45 | -export([start_link/0, 46 | create_table/2, 47 | maybe_create_table/2]). 48 | 49 | %% gen_server callbacks 50 | -export([init/1, 51 | handle_call/3, 52 | handle_cast/2, 53 | handle_info/2, 54 | terminate/2, 55 | code_change/3]). 56 | 57 | %% Unfortunately the `ets' module does not define a type for options, but we 58 | %% can at least insist on a list. 59 | -type ets_options() :: list(). 60 | -type ets_table() :: ets:tab(). 61 | -type create_table_result() :: {ok, ets_table()} | {error, Reason::term()}. 62 | 63 | %%%=================================================================== 64 | %%% API 65 | %%%=================================================================== 66 | 67 | -spec start_link() -> {ok, pid()} | ignore | {error, Reason::term()}. 68 | start_link() -> 69 | gen_server:start_link({local, ?MODULE}, ?MODULE, dict:new(), []). 70 | 71 | %% Creates a new ETS table with the given `Name' and `Options'. 72 | %% Since the table will be owned by the `riak_core_table_owner' process, it 73 | %% should be created with the `public' option so that other processes can read 74 | %% and write data in the table. 75 | -spec create_table(Name::atom(), Options::ets_options()) -> create_table_result(). 76 | create_table(Name, Options) -> 77 | gen_server:call(?MODULE, {create_table, Name, Options}). 78 | 79 | %% Creates a new ETS table with the given `Name' and `Options', if and only if 80 | %% it was not already created previously. 81 | %% Since the table will be owned by the `riak_core_table_owner' process, it 82 | %% should be created with the `public' option so that other processes can read 83 | %% and write data in the table. 84 | -spec maybe_create_table(Name::atom(), Options::ets_options()) -> create_table_result(). 85 | maybe_create_table(Name, Options) -> 86 | gen_server:call(?MODULE, {maybe_create_table, Name, Options}). 87 | 88 | %%%=================================================================== 89 | %%% gen_server callbacks 90 | %%%=================================================================== 91 | 92 | init(State) -> 93 | {ok, State}. 94 | 95 | handle_call({create_table, Name, Options}, _From, State) -> 96 | do_create_table(Name, Options, State); 97 | handle_call({maybe_create_table, Name, Options}, _From, State) -> 98 | case dict:find(Name, State) of 99 | {ok, Table} -> 100 | {reply, {ok, Table}, State}; 101 | error -> 102 | do_create_table(Name, Options, State) 103 | end. 104 | 105 | handle_cast(_Msg, State) -> 106 | {noreply, State}. 107 | 108 | handle_info(_Info, State) -> 109 | {noreply, State}. 110 | 111 | terminate(_Reason, _State) -> 112 | ok. 113 | 114 | code_change(_OldVsn, State, _Extra) -> 115 | {ok, State}. 116 | 117 | %%%=================================================================== 118 | %%% Internal functions 119 | %%%=================================================================== 120 | 121 | do_create_table(Name, Options, State) -> 122 | try 123 | Table = ets:new(Name, Options), 124 | {reply, {ok, Table}, dict:store(Name, Table, State)} 125 | catch 126 | Error -> 127 | {reply, {error, Error}, State} 128 | end. 129 | -------------------------------------------------------------------------------- /src/riak_core_test_util.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riak_test_util: utilities for test scripts 4 | %% 5 | %% Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | %% @doc utilities for test scripts 24 | 25 | -module(riak_core_test_util). 26 | 27 | -ifdef(TEST). 28 | -export([setup_mockring1/0, 29 | fake_ring/2, 30 | stop_pid/1, 31 | wait_for_pid/1, 32 | stop_pid/2, 33 | unlink_named_process/1]). 34 | -include_lib("eunit/include/eunit.hrl"). 35 | 36 | stop_pid(undefined) -> 37 | ok; 38 | stop_pid(Name) when is_atom(Name) -> 39 | stop_pid(whereis(Name)); 40 | stop_pid(Other) when not is_pid(Other) -> 41 | ok; 42 | stop_pid(Pid) -> 43 | stop_pid(Pid, kill). 44 | 45 | stop_pid(Other, _ExitType) when not is_pid(Other) -> 46 | ok; 47 | stop_pid(Pid, ExitType) -> 48 | unlink(Pid), 49 | exit(Pid, ExitType), 50 | ok = wait_for_pid(Pid). 51 | 52 | wait_for_pid(Pid) -> 53 | Mref = erlang:monitor(process, Pid), 54 | receive 55 | {'DOWN', Mref, process, _, _} -> 56 | ok 57 | after 58 | 5000 -> 59 | {error, didnotexit} 60 | end. 61 | 62 | 63 | unlink_named_process(Name) when is_atom(Name) -> 64 | unlink(whereis(Name)). 65 | 66 | setup_mockring1() -> 67 | % requires a running riak_core_ring_manager, in test-mode is ok 68 | Ring0 = riak_core_ring:fresh(16,node()), 69 | Ring1 = riak_core_ring:add_member(node(), Ring0, 'othernode@otherhost'), 70 | Ring2 = riak_core_ring:add_member(node(), Ring1, 'othernode2@otherhost2'), 71 | 72 | Ring3 = lists:foldl(fun(_,R) -> 73 | riak_core_ring:transfer_node( 74 | hd(riak_core_ring:my_indices(R)), 75 | 'othernode@otherhost', R) end, 76 | Ring2,[1,2,3,4,5,6]), 77 | Ring = lists:foldl(fun(_,R) -> 78 | riak_core_ring:transfer_node( 79 | hd(riak_core_ring:my_indices(R)), 80 | 'othernode2@otherhost2', R) end, 81 | Ring3,[1,2,3,4,5,6]), 82 | riak_core_ring_manager:set_ring_global(Ring). 83 | 84 | fake_ring(Size, NumNodes) -> 85 | ManyNodes = [list_to_atom("dev" ++ integer_to_list(X) ++ "@127.0.0.1") 86 | || _ <- lists:seq(0, Size div NumNodes), 87 | X <- lists:seq(1, NumNodes)], 88 | Nodes = lists:sublist(ManyNodes, Size), 89 | Inc = chash:ring_increment(Size), 90 | Indices = lists:seq(0, (Size-1)*Inc, Inc), 91 | Owners = lists:zip(Indices, Nodes), 92 | [Node|OtherNodes] = Nodes, 93 | Ring = riak_core_ring:fresh(Size, Node), 94 | Ring2 = lists:foldl(fun(OtherNode, RingAcc) -> 95 | RingAcc2 = riak_core_ring:add_member(Node, RingAcc, OtherNode), 96 | riak_core_ring:set_member(Node, RingAcc2, OtherNode, 97 | valid, same_vclock) 98 | end, Ring, OtherNodes), 99 | Ring3 = lists:foldl(fun({Idx, Owner}, RingAcc) -> 100 | riak_core_ring:transfer_node(Idx, Owner, RingAcc) 101 | end, Ring2, Owners), 102 | Ring3. 103 | 104 | -endif. %TEST. 105 | -------------------------------------------------------------------------------- /src/riak_core_vnode_proxy_sup.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% Copyright (c) 2007-2011 Basho Technologies, Inc. All Rights Reserved. 3 | %% 4 | %% This file is provided to you under the Apache License, 5 | %% Version 2.0 (the "License"); you may not use this file 6 | %% except in compliance with the License. You may obtain 7 | %% a copy of the License at 8 | %% 9 | %% http://www.apache.org/licenses/LICENSE-2.0 10 | %% 11 | %% Unless required by applicable law or agreed to in writing, 12 | %% software distributed under the License is distributed on an 13 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | %% KIND, either express or implied. See the License for the 15 | %% specific language governing permissions and limitations 16 | %% under the License. 17 | %% 18 | %% ------------------------------------------------------------------- 19 | -module(riak_core_vnode_proxy_sup). 20 | -behaviour(supervisor). 21 | -export([start_link/0, init/1]). 22 | -export([start_proxy/2, stop_proxy/2, start_proxies/1]). 23 | 24 | start_link() -> 25 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 26 | 27 | init([]) -> 28 | %% Populate supervisor list with proxies for already registered vnode 29 | %% modules. Ensures restart of proxies after a crash of this supervisor. 30 | Indices = get_indices(), 31 | VMods = riak_core:vnode_modules(), 32 | Proxies = [proxy_ref(Mod, Index) || {_, Mod} <- VMods, 33 | Index <- Indices], 34 | {ok, {{one_for_one, 5, 10}, Proxies}}. 35 | 36 | start_proxy(Mod, Index) -> 37 | Ref = proxy_ref(Mod, Index), 38 | Pid = case supervisor:start_child(?MODULE, Ref) of 39 | {ok, Child} -> Child; 40 | {error, {already_started, Child}} -> Child 41 | end, 42 | Pid. 43 | 44 | stop_proxy(Mod, Index) -> 45 | _ = supervisor:terminate_child(?MODULE, {Mod, Index}), 46 | _ = supervisor:delete_child(?MODULE, {Mod, Index}), 47 | ok. 48 | 49 | start_proxies(Mod) -> 50 | lager:debug("Starting vnode proxies for: ~p", [Mod]), 51 | Indices = get_indices(), 52 | _ = [start_proxy(Mod, Index) || Index <- Indices], 53 | ok. 54 | 55 | %% @private 56 | proxy_ref(Mod, Index) -> 57 | {{Mod, Index}, {riak_core_vnode_proxy, start_link, [Mod, Index]}, 58 | permanent, 5000, worker, [riak_core_vnode_proxy]}. 59 | 60 | %% @private 61 | get_indices() -> 62 | {ok, Ring} = riak_core_ring_manager:get_raw_ring(), 63 | AllOwners = riak_core_ring:all_owners(Ring), 64 | [Idx || {Idx, _} <- AllOwners]. 65 | -------------------------------------------------------------------------------- /src/riak_core_vnode_sup.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riak_vnode_sup: supervise riak_vnode processes 4 | %% 5 | %% Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | %% @doc supervise riak_vnode processes 24 | 25 | -module(riak_core_vnode_sup). 26 | -behaviour(supervisor). 27 | -export([start_link/0, init/1]). 28 | -export([start_vnode/3]). 29 | 30 | start_vnode(Mod, Index, ForwardTo) when is_integer(Index) -> 31 | supervisor_pre_r14b04:start_child(?MODULE, [Mod, Index, ForwardTo]). 32 | 33 | start_link() -> 34 | %% We use a custom copy of the supervisor module that is expected to be 35 | %% part of R14B04 or R15. This includes the patch that allows 36 | %% simple_one_for_one supervisors to do a controlled shutdown. 37 | %% This is needed because we need to make sure vnode shutdown triggers 38 | %% async worker pool shutdown AND blocks waiting for the worker pool to 39 | %% terminate. 40 | supervisor_pre_r14b04:start_link({local, ?MODULE}, ?MODULE, []). 41 | 42 | %% @private 43 | init([]) -> 44 | {ok, 45 | {{simple_one_for_one, 10, 10}, 46 | [{undefined, 47 | {riak_core_vnode, start_link, []}, 48 | temporary, 300000, worker, dynamic}]}}. 49 | -------------------------------------------------------------------------------- /src/riak_core_vnode_worker.erl: -------------------------------------------------------------------------------- 1 | %% 2 | %% Copyright (c) 2007-2011 Basho Technologies, Inc. All Rights Reserved. 3 | %% 4 | %% This file is provided to you under the Apache License, 5 | %% Version 2.0 (the "License"); you may not use this file 6 | %% except in compliance with the License. You may obtain 7 | %% a copy of the License at 8 | %% 9 | %% http://www.apache.org/licenses/LICENSE-2.0 10 | %% 11 | %% Unless required by applicable law or agreed to in writing, 12 | %% software distributed under the License is distributed on an 13 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | %% KIND, either express or implied. See the License for the 15 | %% specific language governing permissions and limitations 16 | %% under the License. 17 | %% 18 | %% ------------------------------------------------------------------- 19 | -module(riak_core_vnode_worker). 20 | 21 | -behaviour(gen_server). 22 | 23 | -include("riak_core_vnode.hrl"). 24 | 25 | -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, 26 | code_change/3]). 27 | -export([start_link/1, handle_work/3, handle_work/4]). 28 | 29 | -ifdef(PULSE). 30 | -compile(export_all). 31 | -compile({parse_transform, pulse_instrument}). 32 | -compile({pulse_replace_module, [{gen_fsm_compat, pulse_gen_fsm}, 33 | {gen_server, pulse_gen_server}]}). 34 | -endif. 35 | 36 | -type mod_state() :: term(). 37 | 38 | -record(state, { 39 | module :: atom(), 40 | modstate :: mod_state() 41 | }). 42 | 43 | -callback init_worker(partition(), Args :: term(), Props :: [{atom(), term()}]) -> 44 | {ok, mod_state()}. 45 | -callback handle_work(Work :: term(), sender(), mod_state()) -> 46 | {reply, Reply :: term(), mod_state()} | 47 | {noreply, mod_state()}. 48 | 49 | start_link(Args) -> 50 | WorkerMod = proplists:get_value(worker_callback_mod, Args), 51 | [VNodeIndex, WorkerArgs, WorkerProps, Caller] = proplists:get_value(worker_args, Args), 52 | gen_server:start_link(?MODULE, [WorkerMod, VNodeIndex, WorkerArgs, WorkerProps, Caller], []). 53 | 54 | handle_work(Worker, Work, From) -> 55 | handle_work(Worker, Work, From, self()). 56 | 57 | handle_work(Worker, Work, From, Caller) -> 58 | gen_server:cast(Worker, {work, Work, From, Caller}). 59 | 60 | init([Module, VNodeIndex, WorkerArgs, WorkerProps, Caller]) -> 61 | {ok, WorkerState} = Module:init_worker(VNodeIndex, WorkerArgs, WorkerProps), 62 | %% let the pool queue manager know there might be a worker to checkout 63 | gen_fsm_compat:send_all_state_event(Caller, worker_start), 64 | {ok, #state{module=Module, modstate=WorkerState}}. 65 | 66 | handle_call(Event, _From, State) -> 67 | lager:debug("Vnode worker received synchronous event: ~p.", [Event]), 68 | {reply, ok, State}. 69 | 70 | handle_cast({work, Work, WorkFrom, Caller}, 71 | #state{module = Mod, modstate = ModState} = State) -> 72 | NewModState = case Mod:handle_work(Work, WorkFrom, ModState) of 73 | {reply, Reply, NS} -> 74 | riak_core_vnode:reply(WorkFrom, Reply), 75 | NS; 76 | {noreply, NS} -> 77 | NS 78 | end, 79 | %% check the worker back into the pool 80 | gen_fsm_compat:send_all_state_event(Caller, {checkin, self()}), 81 | {noreply, State#state{modstate=NewModState}}; 82 | handle_cast(_Event, State) -> 83 | {noreply, State}. 84 | 85 | handle_info(_Info, State) -> 86 | {noreply, State}. 87 | 88 | terminate(_Reason, _State) -> 89 | ok. 90 | 91 | code_change(_OldVsn, State, _Extra) -> 92 | {ok, State}. 93 | -------------------------------------------------------------------------------- /test/13node_12node_ring.eqc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyorai/riak_core/771add5939d9049cd6fdbcb9a86a7cecafde2e63/test/13node_12node_ring.eqc -------------------------------------------------------------------------------- /test/169_group_join.eqc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyorai/riak_core/771add5939d9049cd6fdbcb9a86a7cecafde2e63/test/169_group_join.eqc -------------------------------------------------------------------------------- /test/648_unbalanced_singly.eqc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyorai/riak_core/771add5939d9049cd6fdbcb9a86a7cecafde2e63/test/648_unbalanced_singly.eqc -------------------------------------------------------------------------------- /test/bucket_eqc_utils.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2007-2016 Basho Technologies, Inc. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | -module(bucket_eqc_utils). 22 | 23 | %% API 24 | -export([per_test_setup/2]). 25 | 26 | 27 | per_test_setup(DefaultBucketProps, TestFun) -> 28 | try 29 | meck:new(riak_core_capability, []), 30 | meck:expect(riak_core_capability, get, 31 | fun({riak_core, bucket_types}) -> true; 32 | (X) -> meck:passthrough([X]) end), 33 | os:cmd("rm -rf ./meta_temp"), 34 | riak_core_test_util:stop_pid(whereis(riak_core_ring_events)), 35 | riak_core_test_util:stop_pid(whereis(riak_core_ring_manager)), 36 | application:set_env(riak_core, claimant_tick, 4294967295), 37 | application:set_env(riak_core, broadcast_lazy_timer, 4294967295), 38 | application:set_env(riak_core, broadcast_exchange_timer, 4294967295), 39 | application:set_env(riak_core, metadata_hashtree_timer, 4294967295), 40 | application:set_env(riak_core, cluster_name, "eqc_test"), 41 | application:set_env(riak_core, default_bucket_props, DefaultBucketProps), 42 | {ok, RingEvents} = riak_core_ring_events:start_link(), 43 | {ok, RingMgr} = riak_core_ring_manager:start_link(test), 44 | {ok, Claimant} = riak_core_claimant:start_link(), 45 | {ok, MetaMgr} = riak_core_metadata_manager:start_link([{data_dir, "./meta_temp"}]), 46 | {ok, Hashtree} = riak_core_metadata_hashtree:start_link("./meta_temp/trees"), 47 | {ok, Broadcast} = riak_core_broadcast:start_link(), 48 | 49 | Results = TestFun(), 50 | 51 | riak_core_test_util:stop_pid(Broadcast), 52 | riak_core_test_util:stop_pid(Hashtree), 53 | riak_core_test_util:stop_pid(MetaMgr), 54 | riak_core_test_util:stop_pid(Claimant), 55 | unlink(RingMgr), 56 | riak_core_ring_manager:stop(), 57 | riak_core_test_util:stop_pid(RingEvents), 58 | Results 59 | after 60 | os:cmd("rm -rf ./meta_temp"), 61 | meck:unload() 62 | end. 63 | -------------------------------------------------------------------------------- /test/claim-statem-leaving-nodes-still-claim.eqc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyorai/riak_core/771add5939d9049cd6fdbcb9a86a7cecafde2e63/test/claim-statem-leaving-nodes-still-claim.eqc -------------------------------------------------------------------------------- /test/claim_32_5_unbalanced.eqc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyorai/riak_core/771add5939d9049cd6fdbcb9a86a7cecafde2e63/test/claim_32_5_unbalanced.eqc -------------------------------------------------------------------------------- /test/claim_simulation.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2011 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | %% Based on an earlier edition provided by Greg Nelson and Ryan Zezeski: 22 | %% https://gist.github.com/992317 23 | 24 | -module(claim_simulation). 25 | -compile(export_all). 26 | 27 | %%-define(SIMULATE,1). 28 | -ifdef(SIMULATE). 29 | -ifdef(TEST). 30 | -include_lib("eunit/include/eunit.hrl"). 31 | 32 | -define(node(N), list_to_atom(lists:flatten(io_lib:format("riak@node~w",[N])))). 33 | -define(get(K, PL, D), proplists:get_value(K, PL, D)). 34 | 35 | basic_test_() -> 36 | {timeout, 60000, [fun basic_default/0, 37 | fun basic_new/0]}. 38 | 39 | basic_default() -> 40 | Opts = [{suffix, "_default"}, 41 | {wc_mf, {riak_core_claim, default_wants_claim}}, 42 | {cc_mf, {riak_core_claim, default_choose_claim}}, 43 | {target_n_val, 4}, 44 | {ring_size, 32}, 45 | {node_count, 8}, 46 | {node_capacity, 24} 47 | ], 48 | run(Opts). 49 | 50 | basic_new() -> 51 | Opts = [{suffix, "_new"}, 52 | {wc_mf, {riak_core_new_claim, new_wants_claim}}, 53 | {cc_mf, {riak_core_new_claim, new_choose_claim}}, 54 | {target_n_val, 4}, 55 | {ring_size, 32}, 56 | {node_count, 8}, 57 | {node_capacity, 24} 58 | ], 59 | run(Opts). 60 | 61 | run(Opts) -> 62 | application:load(riak_core), 63 | 64 | WCMod = ?get(wc_mf, Opts, {riak_core_claim, default_wants_claim}), 65 | CCMod = ?get(cc_mf, Opts, {riak_core_claim, default_choose_claim}), 66 | TargetN = ?get(target_n_val, Opts, 4), 67 | Suffix = ?get(suffix, Opts, ""), 68 | 69 | application:set_env(riak_core, wants_claim_fun, WCMod), 70 | application:set_env(riak_core, choose_claim_fun, CCMod), 71 | application:set_env(riak_core, target_n_val, TargetN), 72 | 73 | RingSize = ?get(ring_size, Opts, 2048), 74 | NodeCount = ?get(node_count, Opts, 100), 75 | NodeCapacity = ?get(node_capacity, Opts, 24), %% in TB 76 | 77 | Ring1 = riak_core_ring:fresh(RingSize, 'riak@node1'), 78 | {Rings, _} = lists:mapfoldl( 79 | fun(N, Prev) -> 80 | Node = ?node(N), 81 | R = riak_core_ring:add_member(Node, Prev, Node), 82 | Next = 83 | riak_core_gossip:claim_until_balanced(R, 84 | Node), 85 | {Next, Next} 86 | end, Ring1, lists:seq(2, NodeCount)), 87 | 88 | Owners1 = riak_core_ring:all_owners(Ring1), 89 | Owners = lists:map(fun riak_core_ring:all_owners/1, Rings), 90 | 91 | {Movers, _} = 92 | lists:mapfoldl( 93 | fun(Curr, Prev) -> 94 | Sum = length(lists:filter(fun not_same_node/1, 95 | lists:zip(Prev, Curr))), 96 | {Sum, Curr} 97 | end, Owners1, Owners), 98 | 99 | MeetTargetN = [riak_core_claim:meets_target_n(R, TargetN) || R <- Rings], 100 | 101 | FName = io_lib:format("/tmp/rings_~w_~w~s.txt", 102 | [RingSize, NodeCount, Suffix]), 103 | {ok, Out} = file:open(FName, [write]), 104 | [print_info(Out, O, N, M, lists:nth(N - 1, MeetTargetN), 105 | RingSize, NodeCapacity) 106 | || {O, M, N} <- lists:zip3(Owners, Movers, lists:seq(2, NodeCount))], 107 | lists:foreach(fun(RingOut) -> 108 | riak_core_ring:pretty_print(RingOut, 109 | [{out, Out}, 110 | {target_n, TargetN}]) 111 | end, [Ring1|Rings]). 112 | 113 | not_same_node({{P,N}, {P,N}}) -> false; 114 | not_same_node({{P,_N}, {P,_M}}) -> true. 115 | 116 | print_info(Out, Owners, NodeCount, PartitionsMoved, MeetsTargetN, 117 | RingSize, NodeCapacity) -> 118 | Expect = round(RingSize / NodeCount), 119 | ActualPercent = 100 * (PartitionsMoved / RingSize), 120 | Terabytes = 0.8 * NodeCount * NodeCapacity * (PartitionsMoved / RingSize), 121 | %% Terabytes = Total Amount of Data * Fraction Moved 122 | 123 | F = fun({_,Node}, Acc) -> 124 | dict:update_counter(Node, 1, Acc) 125 | end, 126 | C = lists:keysort(1, dict:to_list(lists:foldl(F, dict:new(), Owners))), 127 | io:format(Out, 128 | "add node ~p, expect=~p, actual=~p, " 129 | "percent=~p, terabytes=~p, meets_target_n=~p~n" 130 | "counts: ~p~n~n", 131 | [NodeCount, Expect, PartitionsMoved, ActualPercent, 132 | Terabytes, MeetsTargetN,C]). 133 | 134 | -endif. 135 | -endif. 136 | -------------------------------------------------------------------------------- /test/dummy_evt.erl: -------------------------------------------------------------------------------- 1 | 2 | %% ------------------------------------------------------------------- 3 | %% 4 | %% A dummey gen_event listener for testing. 5 | %% 6 | %% Copyright (c) 2015 Basho Technologies, Inc. All Rights Reserved. 7 | %% 8 | %% This file is provided to you under the Apache License, 9 | %% Version 2.0 (the "License"); you may not use this file 10 | %% except in compliance with the License. You may obtain 11 | %% a copy of the License at 12 | %% 13 | %% http://www.apache.org/licenses/LICENSE-2.0 14 | %% 15 | %% Unless required by applicable law or agreed to in writing, 16 | %% software distributed under the License is distributed on an 17 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 18 | %% KIND, either express or implied. See the License for the 19 | %% specific language governing permissions and limitations 20 | %% under the License. 21 | %% 22 | %% ------------------------------------------------------------------- 23 | 24 | -module(dummy_evt). 25 | 26 | -behaviour(gen_event). 27 | 28 | %%% ------------------------------------------------------------------ 29 | %%% gen_event Function Exports 30 | %%% ------------------------------------------------------------------ 31 | 32 | -export([code_change/3]). 33 | -export([handle_call/2]). 34 | -export([handle_event/2]). 35 | -export([handle_info/2]). 36 | -export([init/1]). 37 | -export([terminate/2]). 38 | 39 | -record(state, {}). 40 | 41 | %%% ------------------------------------------------------------------ 42 | %%% gen_event Function Definitions 43 | %%% ------------------------------------------------------------------ 44 | 45 | init(_) -> 46 | {ok, #state{}}. 47 | 48 | handle_event(_Event, State) -> 49 | {ok, State}. 50 | 51 | handle_call(_Request, State) -> 52 | Reply = ok, 53 | {ok, Reply, State}. 54 | 55 | handle_info(_Info, State) -> 56 | {ok, State}. 57 | 58 | terminate(_Reason, _State) -> 59 | ok. 60 | 61 | code_change(_OldVsn, State, _Extra) -> 62 | {ok, State}. 63 | -------------------------------------------------------------------------------- /test/riak_core_base64url_tests.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% Copyright (c) 2016 Basho Technologies, Inc. All Rights Reserved. 3 | %% 4 | %% This file is provided to you under the Apache License, 5 | %% Version 2.0 (the "License"); you may not use this file 6 | %% except in compliance with the License. You may obtain 7 | %% a copy of the License at 8 | %% 9 | %% http://www.apache.org/licenses/LICENSE-2.0 10 | %% 11 | %% Unless required by applicable law or agreed to in writing, 12 | %% software distributed under the License is distributed on an 13 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | %% KIND, either express or implied. See the License for the 15 | %% specific language governing permissions and limitations 16 | %% under the License. 17 | %% ------------------------------------------------------------------- 18 | -module(riak_core_base64url_tests). 19 | 20 | -include_lib("eunit/include/eunit.hrl"). 21 | 22 | -define(URL, "http://example.com/foo?query=thing"). 23 | 24 | string_to_string_test() -> 25 | Encoded = riak_core_base64url:encode_to_string(?URL), 26 | Decoded = riak_core_base64url:decode_to_string(Encoded), 27 | ?assertEqual(?URL, Decoded). 28 | 29 | string_to_binary_test() -> 30 | Encoded = riak_core_base64url:encode(?URL), 31 | Decoded = riak_core_base64url:decode(Encoded), 32 | ?assertEqual(<>, Decoded). 33 | 34 | binary_to_binary_test() -> 35 | Encoded = riak_core_base64url:encode(<>), 36 | Decoded = riak_core_base64url:decode(Encoded), 37 | ?assertEqual(<>, Decoded). 38 | 39 | binary_to_string_test() -> 40 | Encoded = riak_core_base64url:encode_to_string(<>), 41 | Decoded = riak_core_base64url:decode_to_string(Encoded), 42 | ?assertEqual(?URL, Decoded). 43 | -------------------------------------------------------------------------------- /test/riak_core_capability_tests.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riak_core: Core Riak Application 4 | %% 5 | %% Copyright (c) 2007-2017 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | -module(riak_core_capability_tests). 23 | 24 | %% =================================================================== 25 | %% EUnit tests 26 | %% =================================================================== 27 | -ifdef(TEST). 28 | -include_lib("eunit/include/eunit.hrl"). 29 | 30 | basic_test() -> 31 | S1 = riak_core_capability:init_state([]), 32 | 33 | S2 = riak_core_capability:register_capability(n1, 34 | {riak_core, test}, 35 | riak_core_capability:capability_info([x,a,c,y], y, []), 36 | S1), 37 | S3 = riak_core_capability:add_node_capabilities(n2, 38 | [{{riak_core, test}, [a,b,c,y]}], 39 | S2), 40 | S4 = riak_core_capability:negotiate_capabilities(n1, [{riak_core, []}], S3), 41 | ?assertEqual([{{riak_core, test}, a}], riak_core_capability:get_negotiated(S4)), 42 | 43 | S5 = riak_core_capability:negotiate_capabilities(n1, 44 | [{riak_core, [{test, [{prefer, c}]}]}], 45 | S4), 46 | ?assertEqual([{{riak_core, test}, c}], riak_core_capability:get_negotiated(S5)), 47 | 48 | S6 = riak_core_capability:add_node_capabilities(n3, 49 | [{{riak_core, test}, [b]}], 50 | S5), 51 | S7 = riak_core_capability:negotiate_capabilities(n1, [{riak_core, []}], S6), 52 | ?assertEqual([{{riak_core, test}, y}], riak_core_capability:get_negotiated(S7)), 53 | 54 | S8 = riak_core_capability:negotiate_capabilities(n1, 55 | [{riak_core, [{test, [{use, x}]}]}], 56 | S7), 57 | ?assertEqual([{{riak_core, test}, x}], riak_core_capability:get_negotiated(S8)), 58 | ok. 59 | 60 | -endif. 61 | -------------------------------------------------------------------------------- /test/site1-cert.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIICTjCCAg4CCQCSG/O2Rj3DiTAJBgcqhkjOOAQDMA0xCzAJBgNVBAMMAlVTMB4X 3 | DTE4MDUwNDA4MTc0OVoXDTE4MDYwMzA4MTc0OVowDTELMAkGA1UEAwwCVVMwggG2 4 | MIIBKwYHKoZIzjgEATCCAR4CgYEA2SHJry0JUwuxwd3CuyUh5pyo0nj1KNFHoL1E 5 | GkecN7Kh/f6criRyYWUPU0E8auOUFqC2QOfEwFBr5s2yfWTm0uJtmB2lsylJAOdi 6 | w0jgzByBtysw/0YnsV24YSuRAjuCHoVRgs43WtkdCrvw00iEAr83MNUTCeqlFmmV 7 | P5SqntkCFQDaHdXRnQG5gnP/CwDmdKB2HF5N1wKBgFogVjHcuajxYRUGSwkaPnjZ 8 | XI7B99+vxmvMQFPEHL3bhprZx1CwgT52HEMx7DMRc8xt6Q9VqWy7YED1klcrZwWB 9 | u6IWe1xkkXcMsRnKVDkwGm+JqvdzZp7oNdTraYRlWUDLyWN2nU1fJbUyORWp6yhH 10 | t6zJA4JaM25vIJb2SRA9A4GEAAKBgG9j1AClH0dxHmGElQBdnYqnW7W2BraLjIxP 11 | tfMxTivkB0Slg3usd3bvGLtIRtuLw1e0yvk8shw6U2fWV9CO5ucXpRMB0dPklxCf 12 | sCig2uDZ3bmgICtOX9Q30eLCZA+1znw1tH37JsAKsz3B8BQpFfLj/k7cg9aDxqXR 13 | aAm5yE4UMAkGByqGSM44BAMDLwAwLAIUSDGjYt3nj3hU0NvKCC4dOegdgbACFAlk 14 | GtiUqbhZdfeMNN/u9i8gqtxl 15 | -----END CERTIFICATE----- 16 | -------------------------------------------------------------------------------- /test/site1-key.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN DSA PRIVATE KEY----- 2 | MIIBugIBAAKBgQDZIcmvLQlTC7HB3cK7JSHmnKjSePUo0UegvUQaR5w3sqH9/pyu 3 | JHJhZQ9TQTxq45QWoLZA58TAUGvmzbJ9ZObS4m2YHaWzKUkA52LDSODMHIG3KzD/ 4 | RiexXbhhK5ECO4IehVGCzjda2R0Ku/DTSIQCvzcw1RMJ6qUWaZU/lKqe2QIVANod 5 | 1dGdAbmCc/8LAOZ0oHYcXk3XAoGAWiBWMdy5qPFhFQZLCRo+eNlcjsH336/Ga8xA 6 | U8QcvduGmtnHULCBPnYcQzHsMxFzzG3pD1WpbLtgQPWSVytnBYG7ohZ7XGSRdwyx 7 | GcpUOTAab4mq93Nmnug11OtphGVZQMvJY3adTV8ltTI5FanrKEe3rMkDglozbm8g 8 | lvZJED0CgYBvY9QApR9HcR5hhJUAXZ2Kp1u1tga2i4yMT7XzMU4r5AdEpYN7rHd2 9 | 7xi7SEbbi8NXtMr5PLIcOlNn1lfQjubnF6UTAdHT5JcQn7AooNrg2d25oCArTl/U 10 | N9HiwmQPtc58NbR9+ybACrM9wfAUKRXy4/5O3IPWg8al0WgJuchOFAIUaF5O1v/u 11 | BD1d1yz+SfD8ZG/ANrU= 12 | -----END DSA PRIVATE KEY----- 13 | -------------------------------------------------------------------------------- /test/site2-cert.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIICTzCCAg8CCQD29OqoKbLVZDAJBgcqhkjOOAQDMA0xCzAJBgNVBAMMAlVTMB4X 3 | DTE4MDUwNDA4MTc1MFoXDTE4MDYwMzA4MTc1MFowDTELMAkGA1UEAwwCVVMwggG3 4 | MIIBKwYHKoZIzjgEATCCAR4CgYEA1Iq+nn9KPJAIn8QhsGlciTC5m9E3qd/Szp0F 5 | jawMj6I1MZEU2g+P7oI2WD63ACXJ3x6Q+VJ91h+IdsCq4i4r1nxPCLnD7uTtXeNB 6 | xnLzXHTmNurzKadrIwfgz92xsoKZx9ad/EHo1GdWrzs8oYpWiJQi8NW4hcN1R7kv 7 | NkFPcPcCFQDNv9EqG3uOzAQD1iZ6Mtg0qXfhgwKBgA9QY/jLgt1knQRK0kMmp75v 8 | 0REmx8J0FXxWr58WftD/XX2lb3NnW/QG/AAmF7QbGgUIw4X5yIr87XFdWkJXSbsH 9 | pXhVyrMi6UKYDoDZE8Kl5ajiXGyWpa8M/tBtRb28CDFGeCRr6Y/4imWEdl7vrKmX 10 | DSRk7+xpN4eYuCOPZ1OvA4GFAAKBgQCIc+IIDZdFQ0UgtMzvA43ZMV90G05wde// 11 | ykTALfZEnarzkM5eHoJBvt4SzsiG0emPxQhIB7wnwB/GTJqKMMTHSIlEQgc7BJEU 12 | 5M5afyXRiJrR6m3Ot6aXT4T3sTyvHnUJKabvySAXxkfS1c9mdbZ+15fvdNa65jwC 13 | 6WLnjNcjOjAJBgcqhkjOOAQDAy8AMCwCFGASQoCI7nZ42TGffW83q0d9mGNyAhQ/ 14 | U49JgK/63eG4Oqetbgipgbj4tQ== 15 | -----END CERTIFICATE----- 16 | -------------------------------------------------------------------------------- /test/site2-key.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN DSA PRIVATE KEY----- 2 | MIIBvAIBAAKBgQDUir6ef0o8kAifxCGwaVyJMLmb0Tep39LOnQWNrAyPojUxkRTa 3 | D4/ugjZYPrcAJcnfHpD5Un3WH4h2wKriLivWfE8IucPu5O1d40HGcvNcdOY26vMp 4 | p2sjB+DP3bGygpnH1p38QejUZ1avOzyhilaIlCLw1biFw3VHuS82QU9w9wIVAM2/ 5 | 0Sobe47MBAPWJnoy2DSpd+GDAoGAD1Bj+MuC3WSdBErSQyanvm/RESbHwnQVfFav 6 | nxZ+0P9dfaVvc2db9Ab8ACYXtBsaBQjDhfnIivztcV1aQldJuweleFXKsyLpQpgO 7 | gNkTwqXlqOJcbJalrwz+0G1FvbwIMUZ4JGvpj/iKZYR2Xu+sqZcNJGTv7Gk3h5i4 8 | I49nU68CgYEAiHPiCA2XRUNFILTM7wON2TFfdBtOcHXv/8pEwC32RJ2q85DOXh6C 9 | Qb7eEs7IhtHpj8UISAe8J8AfxkyaijDEx0iJREIHOwSRFOTOWn8l0Yia0eptzrem 10 | l0+E97E8rx51CSmm78kgF8ZH0tXPZnW2fteX73TWuuY8Auli54zXIzoCFQCinvk+ 11 | V7tjo8Tpl55IrKEy4yGhHg== 12 | -----END DSA PRIVATE KEY----- 13 | -------------------------------------------------------------------------------- /test/sync_command_test.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2007-2011 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | -module(sync_command_test). 21 | 22 | -include_lib("eunit/include/eunit.hrl"). 23 | -include_lib("riak_core_vnode.hrl"). 24 | 25 | sync_test_() -> 26 | {foreach, 27 | fun setup_simple/0, 28 | fun stop_servers/1, 29 | [ {<<"Assert ok throw">>, 30 | fun() -> 31 | ?assertEqual(ok, mock_vnode:sync_error({0, node()}, goodthrow)) 32 | end}, 33 | {<<"Assert error throw">>, 34 | fun() -> 35 | ?assertEqual({error, terrible}, mock_vnode:sync_error({0, node()}, badthrow)) 36 | end}, 37 | {<<"Assert sync error">>, 38 | fun() -> 39 | ?assertError(core_breach, mock_vnode:sync_error({0, node()}, error)) 40 | end}, 41 | {<<"Assert sync exit">>, 42 | fun() -> 43 | ?assertError(core_breach, mock_vnode:sync_error({0, node()}, exit)) 44 | end}, 45 | {<<"Assert non-blocking sync error">>, 46 | fun() -> 47 | ?assertError(core_breach, mock_vnode:spawn_error({0, node()}, error)) 48 | end}, 49 | {<<"Assert non-blocking sync exit">>, 50 | fun() -> 51 | ?assertError(core_breach, mock_vnode:spawn_error({0, node()}, exit)) 52 | end} 53 | ] 54 | }. 55 | 56 | 57 | setup_simple() -> 58 | stop_servers(self()), 59 | Vars = [{ring_creation_size, 8}, 60 | {ring_state_dir, ""}, 61 | %% Don't allow rolling start of vnodes as it will cause a 62 | %% race condition with `all_nodes'. 63 | {core_vnode_eqc_pool_size, 0}, 64 | {vnode_rolling_start, 0}], 65 | error_logger:tty(false), 66 | _ = [begin 67 | Old = app_helper:get_env(riak_core, AppKey), 68 | ok = application:set_env(riak_core, AppKey, Val), 69 | {AppKey, Old} 70 | end || {AppKey, Val} <- Vars], 71 | exometer:start(), 72 | riak_core_ring_events:start_link(), 73 | riak_core_ring_manager:start_link(test), 74 | riak_core_vnode_proxy_sup:start_link(), 75 | 76 | {ok, _Sup} = riak_core_vnode_sup:start_link(), 77 | {ok, _} = riak_core_vnode_manager:start_link(), 78 | {ok, _VMaster} = riak_core_vnode_master:start_link(mock_vnode), 79 | 80 | 81 | ok = mock_vnode:start_vnode(0), 82 | %% NOTE: The return value from this call is currently not being 83 | %% used. This call is made to ensure that all msgs sent to 84 | %% the vnode mgr before this call have been handled. This 85 | %% guarantees that the vnode mgr ets tab is up-to-date 86 | 87 | riak_core:register([{vnode_module, mock_vnode}]). 88 | 89 | 90 | stop_servers(_Pid) -> 91 | %% Make sure VMaster is killed before sup as start_vnode is a cast 92 | %% and there may be a pending request to start the vnode. 93 | riak_core_test_util:stop_pid(mock_vnode_master), 94 | riak_core_test_util:stop_pid(riak_core_vnode_manager), 95 | riak_core_test_util:stop_pid(riak_core_ring_events), 96 | riak_core_test_util:stop_pid(riak_core_vnode_sup), 97 | riak_core_test_util:stop_pid(riak_core_ring_manager), 98 | application:stop(exometer), 99 | application:stop(lager), 100 | application:stop(goldrush). 101 | -------------------------------------------------------------------------------- /test/test_guarded_event_handler.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% test_guarded_event_handler 4 | %% 5 | %% Copyright (c) 2007-2011 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | -module(test_guarded_event_handler). 23 | -behaviour(gen_event). 24 | -export([start_link/0]). 25 | -export([init/1, handle_event/2, handle_call/2, 26 | handle_info/2, terminate/2, code_change/3]). 27 | -export([get_events/0]). 28 | -record(state, {events=[]}). 29 | 30 | -include_lib("eunit/include/eunit.hrl"). 31 | 32 | start_link() -> 33 | gen_event:start_link({local, ?MODULE}). 34 | 35 | get_events() -> 36 | gen_event:call(?MODULE, ?MODULE, get_events). 37 | 38 | init([]) -> 39 | {ok, #state{}}. 40 | 41 | handle_event({event, E}, State=#state{events=Events}) -> 42 | {ok, State#state{events=[E|Events]}}; 43 | handle_event(crash, State) -> 44 | exit(crash), 45 | {ok, State}. 46 | 47 | handle_call(get_events, State) -> 48 | {ok, State#state.events, State}. 49 | 50 | handle_info(_Info, State) -> 51 | {ok, State}. 52 | 53 | terminate(Reason, _State) -> 54 | Reason. 55 | 56 | code_change(_OldVsn, State, _Extra) -> 57 | {ok, State}. 58 | 59 | -ifdef(TEST). 60 | 61 | guarded_handler_test_() -> 62 | { setup, local, 63 | fun setup/0, 64 | fun cleanup/1, 65 | [ 66 | fun guarded_handler_test_case/0 67 | ] 68 | }. 69 | 70 | setup() -> 71 | riak_core_eventhandler_sup:start_link(), 72 | ?MODULE:start_link(). 73 | 74 | cleanup(_Pid) -> 75 | %% ARG: ugly hack to not die when the supervisor exits. 76 | process_flag(trap_exit, true), 77 | gen_event:stop(?MODULE). 78 | 79 | wait_for_exitfun() -> 80 | receive 81 | {?MODULE, {'EXIT', crash}} -> 82 | ok 83 | after 5000 -> 84 | fail 85 | end. 86 | 87 | guarded_handler_test_case() -> 88 | Self = self(), 89 | F = fun(Handler, Reason) -> 90 | Self ! {Handler, Reason} 91 | end, 92 | riak_core:add_guarded_event_handler(?MODULE, ?MODULE, [], F), 93 | gen_event:notify(?MODULE, {event, foo}), 94 | ?assertEqual(?MODULE:get_events(), [foo]), 95 | gen_event:notify(?MODULE, crash), 96 | ?assertEqual(wait_for_exitfun(), ok), 97 | wait_for_handler(?MODULE, 1000, 100), 98 | gen_event:notify(?MODULE, {event, baz}), 99 | ?assertEqual(?MODULE:get_events(), [baz]), 100 | ?assertEqual(riak_core:delete_guarded_event_handler(?MODULE,?MODULE,quux), quux), 101 | ?assertNot(lists:member(?MODULE, gen_event:which_handlers(?MODULE))), 102 | ?assertEqual([], supervisor:which_children(riak_core_eventhandler_sup)). 103 | 104 | wait_for_handler(_, 0, _) -> 105 | fail; 106 | wait_for_handler(Name, Count, Sleep) -> 107 | case lists:member(Name, gen_event:which_handlers(?MODULE)) of 108 | true -> ok; 109 | false -> 110 | timer:sleep(Sleep), 111 | wait_for_handler(Name, Count - 1, Sleep); 112 | _ -> 113 | ok 114 | end. 115 | 116 | -endif. 117 | -------------------------------------------------------------------------------- /test/worker_pool_pulse.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2013 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | %% @doc Test riak_core_vnode_worker_pool's interaction with poolboy 22 | %% under PULSE. This requires that riak_core, poolboy, and this module 23 | %% be compiled with the 'PULSE' macro defined. 24 | -module(worker_pool_pulse). 25 | 26 | -behaviour(riak_core_vnode_worker). 27 | -ifdef(EQC). 28 | -include_lib("eqc/include/eqc.hrl"). 29 | -endif. 30 | -include_lib("eunit/include/eunit.hrl"). 31 | 32 | %% riak_core_vnode_worker behavior 33 | -export([init_worker/3, handle_work/3]). 34 | %% console debugging convenience 35 | -compile(export_all). 36 | 37 | -ifdef(PULSE). 38 | -include_lib("pulse/include/pulse.hrl"). 39 | %% have to transform the 'receive' of the work results 40 | -compile({parse_transform, pulse_instrument}). 41 | %% don't trasnform toplevel test functions 42 | -compile({pulse_skip,[{prop_any_pool,0}, 43 | {prop_small_pool,0}, 44 | {pool_test_,0}]}). 45 | -endif. 46 | 47 | %%% Worker Definition - does nothing but reply with what it is given 48 | 49 | init_worker(_VnodeIndex, Noreply, _WorkerProps) -> 50 | {ok, Noreply}. 51 | 52 | handle_work(die, _From, _State) -> 53 | exit(test_die); 54 | handle_work(Work, _From, State) -> 55 | {reply, Work, State}. 56 | 57 | %% none of these tests make sense if PULSE is not used 58 | -ifdef(PULSE). 59 | 60 | %% @doc Any amount of work should complete through any size pool. 61 | prop_any_pool() -> 62 | ?FORALL({Seed, ExtraWork, WorkList}, 63 | {pulse:seed(), 64 | frequency([{10,true},{1,false}]), 65 | list(frequency([{10,nat()}, {1,die}]))}, 66 | aggregate([{{extra_work, 67 | pool_size(ExtraWork, WorkList) < length(WorkList) }, 68 | {deaths, lists:member(die, WorkList)}}], 69 | ?WHENFAIL( 70 | io:format(user, 71 | "PoolSize: ~b~n" 72 | "WorkList: ~p~n" 73 | "Schedule: ~p~n", 74 | [pool_size(ExtraWork, WorkList), 75 | WorkList, 76 | pulse:get_schedule()]), 77 | begin 78 | PoolSize = pool_size(ExtraWork, WorkList), 79 | true == all_work_gets_done(Seed, PoolSize, WorkList) 80 | end))). 81 | 82 | pool_size(false, WorkList) -> 83 | length(WorkList); 84 | pool_size(true, WorkList) -> 85 | case length(WorkList) div 2 of 86 | 0 -> 87 | 1; 88 | Size -> 89 | Size 90 | end. 91 | 92 | %% @doc Minimal case for the issue this test was created to probe: one 93 | %% worker, two inputs. 94 | prop_small_pool() -> 95 | ?PULSE(Result, all_work_gets_done(1, [1,2]), true == Result). 96 | 97 | all_work_gets_done(Seed, PoolSize, WorkList) -> 98 | pulse:run_with_seed( 99 | fun() -> all_work_gets_done(PoolSize, WorkList) end, 100 | Seed). 101 | 102 | all_work_gets_done(PoolSize, WorkList) -> 103 | %% get the pool up 104 | {ok, Pool} = riak_core_vnode_worker_pool:start_link( 105 | ?MODULE, PoolSize, 10, false, []), 106 | 107 | %% send all the work 108 | [ riak_core_vnode_worker_pool:handle_work( 109 | Pool, W, {raw, N, self()}) 110 | || {N, W} <- lists:zip(lists:seq(1, length(WorkList)), WorkList) ], 111 | 112 | %% wait for all the work 113 | Results = [ receive {N, _} -> ok end 114 | || N <- lists:seq(1, length(WorkList)) ], 115 | riak_core_vnode_worker_pool:stop(Pool, normal), 116 | 117 | %% check that we got a response for every piece of work 118 | %% TODO: actually not needed, since the bug is deadlock, and will 119 | %% thus never get here 120 | length(Results) == length(WorkList). 121 | 122 | pool_test_() -> 123 | {setup, 124 | fun() -> 125 | error_logger:tty(false), 126 | pulse:start() 127 | end, 128 | fun(_) -> 129 | pulse:stop(), 130 | error_logger:tty(true) 131 | end, 132 | [ 133 | %% not necessary to run both tests here, but why not anyway? 134 | ?_assert(eqc:quickcheck(prop_small_pool())), 135 | ?_assert(eqc:quickcheck(prop_any_pool())) 136 | ] 137 | }. 138 | 139 | -endif. 140 | -------------------------------------------------------------------------------- /test/worker_pool_test.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2007-2011 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | -module(worker_pool_test). 21 | 22 | -behaviour(riak_core_vnode_worker). 23 | -include_lib("eunit/include/eunit.hrl"). 24 | 25 | -export([init_worker/3, handle_work/3]). 26 | 27 | init_worker(_VnodeIndex, Noreply, _WorkerProps) -> 28 | {ok, Noreply}. 29 | 30 | handle_work(Work, From, true = State) -> 31 | Work(), 32 | riak_core_vnode:reply(From, ok), 33 | {noreply, State}; 34 | handle_work(Work, _From, false = State) -> 35 | Work(), 36 | {reply, ok, State}. 37 | 38 | -ifdef(TEST). 39 | 40 | receive_result(N) -> 41 | receive 42 | {N, ok} when N rem 2 /= 0 -> 43 | true; 44 | {N, {error, {worker_crash, _, _}}} when N rem 2 == 0 -> 45 | true 46 | after 47 | 0 -> 48 | timeout 49 | end. 50 | 51 | simple_worker_pool() -> 52 | {ok, Pool} = riak_core_vnode_worker_pool:start_link(?MODULE, 3, 10, false, []), 53 | [ riak_core_vnode_worker_pool:handle_work(Pool, fun() -> 54 | timer:sleep(100), 55 | 1/(N rem 2) 56 | end, 57 | {raw, N, self()}) 58 | || N <- lists:seq(1, 10)], 59 | 60 | timer:sleep(1200), 61 | 62 | %% make sure we got all the expected responses 63 | 64 | [ ?assertEqual(true, receive_result(N)) || N <- lists:seq(1, 10)], 65 | unlink(Pool), 66 | riak_core_vnode_worker_pool:stop(Pool, normal). 67 | 68 | simple_noreply_worker_pool() -> 69 | {ok, Pool} = riak_core_vnode_worker_pool:start_link(?MODULE, 3, 10, true, []), 70 | [ riak_core_vnode_worker_pool:handle_work(Pool, fun() -> 71 | timer:sleep(100), 72 | 1/(N rem 2) 73 | end, 74 | {raw, N, self()}) 75 | || N <- lists:seq(1, 10)], 76 | 77 | timer:sleep(1200), 78 | 79 | %% make sure we got all the expected responses 80 | 81 | [ ?assertEqual(true, receive_result(N)) || N <- lists:seq(1, 10)], 82 | unlink(Pool), 83 | riak_core_vnode_worker_pool:stop(Pool, normal). 84 | 85 | 86 | pool_test_() -> 87 | {setup, 88 | fun() -> 89 | error_logger:tty(false) 90 | end, 91 | fun(_) -> 92 | error_logger:tty(true) 93 | end, 94 | [ 95 | fun simple_worker_pool/0, 96 | fun simple_noreply_worker_pool/0 97 | ] 98 | }. 99 | 100 | -endif. 101 | -------------------------------------------------------------------------------- /tools.mk: -------------------------------------------------------------------------------- 1 | REBAR ?= ./rebar3 2 | 3 | .PHONY: compile-no-deps test docs xref dialyzer-run dialyzer-quick dialyzer \ 4 | cleanplt 5 | 6 | compile-no-deps: 7 | ${REBAR} compile 8 | 9 | test: compile 10 | ${REBAR} eunit 11 | 12 | docs: 13 | ${REBAR} doc 14 | 15 | xref: compile 16 | ${REBAR} xre 17 | 18 | PLT ?= $(HOME)/.combo_dialyzer_plt 19 | LOCAL_PLT = .local_dialyzer_plt 20 | DIALYZER_FLAGS ?= -Wunmatched_returns 21 | 22 | ${PLT}: compile 23 | @if [ -f $(PLT) ]; then \ 24 | dialyzer --check_plt --plt $(PLT) --apps $(DIALYZER_APPS) && \ 25 | dialyzer --add_to_plt --plt $(PLT) --output_plt $(PLT) --apps $(DIALYZER_APPS) ; test $$? -ne 1; \ 26 | else \ 27 | dialyzer --build_plt --output_plt $(PLT) --apps $(DIALYZER_APPS); test $$? -ne 1; \ 28 | fi 29 | 30 | ${LOCAL_PLT}: compile 31 | @if [ -d deps ]; then \ 32 | if [ -f $(LOCAL_PLT) ]; then \ 33 | dialyzer --check_plt --plt $(LOCAL_PLT) deps/*/ebin && \ 34 | dialyzer --add_to_plt --plt $(LOCAL_PLT) --output_plt $(LOCAL_PLT) deps/*/ebin ; test $$? -ne 1; \ 35 | else \ 36 | dialyzer --build_plt --output_plt $(LOCAL_PLT) deps/*/ebin ; test $$? -ne 1; \ 37 | fi \ 38 | fi 39 | 40 | dialyzer-run: 41 | @echo "==> $(shell basename $(shell pwd)) (dialyzer)" 42 | # The bulk of the code below deals with the dialyzer.ignore-warnings file 43 | # which contains strings to ignore if output by dialyzer. 44 | # Typically the strings include line numbers. Using them exactly is hard 45 | # to maintain as the code changes. This approach instead ignores the line 46 | # numbers, but takes into account the number of times a string is listed 47 | # for a given file. So if one string is listed once, for example, and it 48 | # appears twice in the warnings, the user is alerted. It is possible but 49 | # unlikely that this approach could mask a warning if one ignored warning 50 | # is removed and two warnings of the same kind appear in the file, for 51 | # example. But it is a trade-off that seems worth it. 52 | # Details of the cryptic commands: 53 | # - Remove line numbers from dialyzer.ignore-warnings 54 | # - Pre-pend duplicate count to each warning with sort | uniq -c 55 | # - Remove annoying white space around duplicate count 56 | # - Save in dialyer.ignore-warnings.tmp 57 | # - Do the same to dialyzer_warnings 58 | # - Remove matches from dialyzer.ignore-warnings.tmp from output 59 | # - Remove duplicate count 60 | # - Escape regex special chars to use lines as regex patterns 61 | # - Add pattern to match any line number (file.erl:\d+:) 62 | # - Anchor to match the entire line (^entire line$) 63 | # - Save in dialyzer_unhandled_warnings 64 | # - Output matches for those patterns found in the original warnings 65 | @if [ -f $(LOCAL_PLT) ]; then \ 66 | PLTS="$(PLT) $(LOCAL_PLT)"; \ 67 | else \ 68 | PLTS=$(PLT); \ 69 | fi; \ 70 | if [ -f dialyzer.ignore-warnings ]; then \ 71 | if [ $$(grep -cvE '[^[:space:]]' dialyzer.ignore-warnings) -ne 0 ]; then \ 72 | echo "ERROR: dialyzer.ignore-warnings contains a blank/empty line, this will match all messages!"; \ 73 | exit 1; \ 74 | fi; \ 75 | dialyzer $(DIALYZER_FLAGS) --plts $${PLTS} -c ebin > dialyzer_warnings ; \ 76 | cat dialyzer.ignore-warnings \ 77 | | sed -E 's/^([^:]+:)[^:]+:/\1/' \ 78 | | sort \ 79 | | uniq -c \ 80 | | sed -E '/.*\.erl: /!s/^[[:space:]]*[0-9]+[[:space:]]*//' \ 81 | > dialyzer.ignore-warnings.tmp ; \ 82 | egrep -v "^[[:space:]]*(done|Checking|Proceeding|Compiling)" dialyzer_warnings \ 83 | | sed -E 's/^([^:]+:)[^:]+:/\1/' \ 84 | | sort \ 85 | | uniq -c \ 86 | | sed -E '/.*\.erl: /!s/^[[:space:]]*[0-9]+[[:space:]]*//' \ 87 | | grep -F -f dialyzer.ignore-warnings.tmp -v \ 88 | | sed -E 's/^[[:space:]]*[0-9]+[[:space:]]*//' \ 89 | | sed -E 's/([]\^:+?|()*.$${}\[])/\\\1/g' \ 90 | | sed -E 's/(\\\.erl\\\:)/\1\\d+:/g' \ 91 | | sed -E 's/^(.*)$$/^\1$$/g' \ 92 | > dialyzer_unhandled_warnings ; \ 93 | rm dialyzer.ignore-warnings.tmp; \ 94 | if [ $$(cat dialyzer_unhandled_warnings | wc -l) -gt 0 ]; then \ 95 | egrep -f dialyzer_unhandled_warnings dialyzer_warnings ; \ 96 | found_warnings=1; \ 97 | fi; \ 98 | [ "$$found_warnings" != 1 ] ; \ 99 | else \ 100 | dialyzer $(DIALYZER_FLAGS) --plts $${PLTS} -c ebin; \ 101 | fi 102 | 103 | dialyzer-quick: compile-no-deps dialyzer-run 104 | 105 | dialyzer: ${PLT} ${LOCAL_PLT} dialyzer-run 106 | 107 | cleanplt: 108 | @echo 109 | @echo "Are you sure? It takes several minutes to re-build." 110 | @echo Deleting $(PLT) and $(LOCAL_PLT) in 5 seconds. 111 | @echo 112 | sleep 5 113 | rm $(PLT) 114 | rm $(LOCAL_PLT) 115 | 116 | --------------------------------------------------------------------------------