├── .gitignore ├── .manifest ├── .travis.yml ├── DEVELOPMENT.md ├── LICENSE ├── Makefile ├── README.md ├── dialyzer.ignore-warnings ├── doc └── overview.edoc ├── priv ├── ForkMe_Blk.png ├── doctorbasho.jpg ├── edoc.css └── index.html ├── rebar ├── rebar.config └── src ├── riaknostic.app.src ├── riaknostic.erl ├── riaknostic_check.erl ├── riaknostic_check_disk.erl ├── riaknostic_check_dumps.erl ├── riaknostic_check_memory_use.erl ├── riaknostic_check_monitors.erl ├── riaknostic_check_nodes_connected.erl ├── riaknostic_check_ring_membership.erl ├── riaknostic_check_ring_preflists.erl ├── riaknostic_check_ring_size.erl ├── riaknostic_check_search.erl ├── riaknostic_check_strong_consistency.erl ├── riaknostic_config.erl ├── riaknostic_node.erl └── riaknostic_util.erl /.gitignore: -------------------------------------------------------------------------------- 1 | doc/ 2 | deps/ 3 | ebin/* 4 | log/ 5 | edoc/ 6 | index.html 7 | riaknostic 8 | *.png 9 | pkg/ 10 | erl_crash.dump 11 | .eunit/ 12 | *~ 13 | #*# -------------------------------------------------------------------------------- /.manifest: -------------------------------------------------------------------------------- 1 | src 2 | riaknostic 3 | doc 4 | LICENSE 5 | README.md 6 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: erlang 2 | notifications: 3 | disabled: true 4 | env: 5 | - R15B 6 | - R14B04 7 | - R14B03 8 | - R14B02 9 | -------------------------------------------------------------------------------- /DEVELOPMENT.md: -------------------------------------------------------------------------------- 1 | # Riaknostic Development 2 | 3 | Riaknostic requires a sane GNU build system and a recent version of 4 | Erlang. It has `lager` and `getopt` as dependencies, so those must be 5 | compatible with your version of Erlang. Release versions are currently 6 | built with Erlang version R14B03, while development versions are targeted at Erlang version R14B04. 7 | 8 | See the `rebar.config` file for more details. 9 | 10 | To build Riaknostic, simply run `make`: 11 | 12 | ```bash 13 | $ make 14 | ./rebar get-deps 15 | ==> riaknostic (get-deps) 16 | Pulling lager from {git,"git://github.com/basho/lager",{branch,"master"}} 17 | Cloning into lager... 18 | Pulling getopt from {git,"git://github.com/jcomellas/getopt.git","2981dfe"} 19 | Cloning into getopt... 20 | ==> lager (get-deps) 21 | ==> getopt (get-deps) 22 | ./rebar compile 23 | ==> lager (compile) 24 | Compiled src/lager_util.erl 25 | Compiled src/lager_transform.erl 26 | Compiled src/lager_sup.erl 27 | Compiled src/lager_mochiglobal.erl 28 | Compiled src/lager_stdlib.erl 29 | Compiled src/lager_handler_watcher_sup.erl 30 | Compiled src/lager_handler_watcher.erl 31 | Compiled src/lager_trunc_io.erl 32 | Compiled src/lager_crash_log.erl 33 | Compiled src/lager_file_backend.erl 34 | Compiled src/lager_app.erl 35 | Compiled src/lager.erl 36 | Compiled src/lager_console_backend.erl 37 | Compiled src/lager_format.erl 38 | Compiled src/error_logger_lager_h.erl 39 | ==> getopt (compile) 40 | Compiled src/getopt.erl 41 | ==> riaknostic (compile) 42 | Compiled src/riaknostic_check.erl 43 | Compiled src/riaknostic_util.erl 44 | Compiled src/riaknostic_node.erl 45 | Compiled src/riaknostic_check_ring_size.erl 46 | Compiled src/riaknostic_check_ring_membership.erl 47 | Compiled src/riaknostic_config.erl 48 | Compiled src/riaknostic_check_memory_use.erl 49 | Compiled src/riaknostic_check_nodes_connected.erl 50 | Compiled src/riaknostic_check_dumps.erl 51 | Compiled src/riaknostic.erl 52 | Compiled src/riaknostic_check_disk.erl 53 | ./rebar escriptize 54 | ==> lager (escriptize) 55 | ==> getopt (escriptize) 56 | ==> riaknostic (escriptize) 57 | ``` 58 | 59 | Now you can invoke the script manually via the below command: 60 | 61 | ```bash 62 | $ ./riaknostic --etc ~/code/riak/rel/riak/etc --base ~/code/riak/rel/riak --user `whoami` [other options] 63 | ``` 64 | 65 | To generate the edoc reference, use `make docs` and then open the 66 | `doc/index.html` file in your browser. Detailed discussion of the 67 | internal APIs that you can use in developing new diagnostics is found 68 | in the edocs. 69 | 70 | ## Contributing 71 | 72 | Have an idea for a diagnostic? Want to improve the way Riaknostic works? Fork the [github repository](https://github.com/basho/riaknostic) and send us a pull-request with your changes! The code is documented with `edoc`, so give the [API Docs](http://riaknostic.basho.com/edoc/index.html) a read before you contribute. 73 | 74 | ### Developing for Riaknostic Without a Riak Instance 75 | 76 | If you want to run the `riaknostic` script while developing, and you don't have it hooked up to your local Riak, you can invoke it directly like so: 77 | 78 | ```bash 79 | ./riaknostic --etc ~/code/riak/rel/riak/etc --base ~/code/riak/rel/riak --user `whoami` [other options] 80 | ``` 81 | 82 | The extra options are usually assigned by the `riak-admin` script for you, but here's how to set them: 83 | 84 | * `--etc`: Where your Riak configuration directory is, in the example above it's in the generated directory of a source checkout of Riak. 85 | * `--base`: The "base" directory of Riak, usually the root of the generated directory or `/usr/lib/riak` on Linux, for example. Scan the `riak-admin` script for how the `RUNNER_BASE_DIR` variable is assigned on your platform. 86 | * `--user`: What user/UID the Riak node runs as. In a source checkout, it's the current user, on most systems, it's `riak`. -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: rel stagedevrel deps test 2 | 3 | all: deps compile 4 | 5 | compile: 6 | ./rebar compile 7 | 8 | deps: 9 | ./rebar get-deps 10 | 11 | clean: 12 | ./rebar clean 13 | 14 | distclean: clean 15 | ./rebar delete-deps 16 | 17 | test: 18 | ./rebar compile eunit 19 | 20 | escriptize: 21 | ./rebar escriptize 22 | 23 | ## 24 | ## Doc targets 25 | ## 26 | docs: 27 | ./rebar doc skip_deps=true 28 | 29 | pages: docs 30 | cp priv/index.html doc/_index.html 31 | cp priv/ForkMe_Blk.png doc/ 32 | cp priv/*.jpg doc/ 33 | git checkout gh-pages 34 | mv doc/_index.html ./index.html 35 | mv doc/ForkMe_Blk.png . 36 | mv doc/*.jpg . 37 | rm -rf edoc/* 38 | cp -R doc/* edoc/ 39 | git add . 40 | git add -u 41 | git commit 42 | git push origin gh-pages 43 | git checkout master 44 | 45 | ## 46 | ## Release targets 47 | ## 48 | VSN = `grep vsn src/riaknostic.app.src | cut -f 2 -d "\""` 49 | 50 | package: all docs 51 | @mkdir -p pkg/riaknostic 52 | @rm -rf pkg/riaknostic/* 53 | @cat .manifest | xargs -n 1 -I % cp -R % pkg/riaknostic/. 54 | @tar -czf pkg/riaknostic-$(VSN).tar.gz -C pkg riaknostic 55 | 56 | ## 57 | ## Dialyzer targets 58 | ## 59 | APPS = kernel stdlib sasl erts ssl tools os_mon runtime_tools crypto inets \ 60 | xmerl webtool snmp public_key mnesia eunit syntax_tools compiler 61 | COMBO_PLT = $(HOME)/.riak_combo_dialyzer_plt 62 | 63 | check_plt: compile 64 | dialyzer --check_plt --plt $(COMBO_PLT) --apps $(APPS) 65 | 66 | build_plt: compile 67 | dialyzer --build_plt --output_plt $(COMBO_PLT) --apps $(APPS) 68 | 69 | dialyzer: compile 70 | @echo 71 | @echo Use "'make check_plt'" to check PLT prior to using this target. 72 | @echo Use "'make build_plt'" to build PLT prior to using this target. 73 | @echo 74 | @sleep 1 75 | dialyzer -Wno_return --plt $(COMBO_PLT) ebin deps/*/ebin | \ 76 | fgrep -v -f ./dialyzer.ignore-warnings 77 | 78 | cleanplt: 79 | @echo 80 | @echo "Are you sure? It takes about 1/2 hour to re-build." 81 | @echo Deleting $(COMBO_PLT) in 5 seconds. 82 | @echo 83 | sleep 5 84 | rm $(COMBO_PLT) 85 | 86 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Riaknostic [![Build Status](https://secure.travis-ci.org/basho/riaknostic.png?branch=master)](http://travis-ci.org/basho/riaknostic) 2 | 3 | `riaknostic` is an escript and set of tools that diagnoses common problems which could affect a Riak node or cluster. When experiencing any problem with Riak, `riaknostic` should be the first thing run during troubleshooting. The tool is integrated with Riak via the `riak-admin` script. 4 | 5 | ## Overview 6 | 7 | To diagnose problems with Riak, Riaknostic uses a series of checks which are derived from the experience of the Basho Client Services Team as well as numerous public discussions on the mailing list, IRC room, and other online media. 8 | 9 | Here is a basic example of using `riaknostic` followed immediately by the command's output: 10 | 11 | ```bash 12 | $ riak-admin diag 13 | 15:34:52.736 [warning] Riak crashed at Wed, 07 Dec 2011 21:47:50 GMT, leaving 14 | crash dump in /srv/riak/log/erl_crash.dump. Please inspect or remove the file. 15 | 15:34:52.736 [notice] Data directory /srv/riak/data/bitcask is not mounted with 'noatime'. Please remount its disk with the 'noatime' flag to improve 16 | performance. 17 | ``` 18 | 19 | As shown in the above output, Riaknostic tells us about two problems right away. First, an Erlang crash dump is present, indicating that Riak has experienced a crash. Second, a performance problem is mentioned (disk mounted without `noatime` argument)along with a helpful tip to resolve the issue. 20 | 21 | ## Installation 22 | 23 | **Important**: If you are running Riak v1.3.0 or greater, you already have Riaknostic, so you can skip to the **Usage** section below. 24 | 25 | Riaknostic depends on features introduced by Erlang version R14B04, so verify that you've installed this version of Erlang before proceeding with installation. 26 | 27 | To install `riaknostic`, download the latest package version, and extract it within the directory shown for your operating system in the following table: 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 55 | 56 | 57 |
PlatformDirectory
Linux (Redhat, CentOS, Debian, Ubuntu)/usr/lib/riak/lib
Linux (Fedora)/usr/lib64/riak/lib
Solaris, OpenSolaris/opt/riak/lib
SmartOS (Joyent)/opt/local/lib/riak/lib
Mac OS/X or Self-built$RIAK/lib 53 | (where $RIAK=rel/riak for source installs, 54 | or the directory where you unpacked the package)
58 | 59 | An example Riaknostic installation for Linux looks like this: 60 | 61 | ```bash 62 | wget https://github.com/basho/riaknostic/downloads/riaknostic-1.0.2.tar.gz -P /tmp 63 | cd /usr/lib/riak/lib 64 | sudo tar xzvf /tmp/riaknostic-1.0.2.tar.gz 65 | ``` 66 | 67 | The package will expand to a `riaknostic/` directory which contains the `riaknostic` script, source code in the `src/` directory, and documentation. 68 | 69 | Now try it out! 70 | 71 | ## Usage 72 | 73 | For most cases, you can just run the `riak-admin diag` command as given at the top of this README. However, sometimes you might want to know some extra detail or run only specific checks. For that, there are command-line options. Execute `riaknostic --help` to learn more about these options: 74 | 75 | ```bash 76 | riak-admin diag --help 77 | Usage: riak-admin diag [-d ] [-l] [-h] [check_name ...] 78 | 79 | -d, --level Minimum message severity level (default: notice) 80 | -l, --list Describe available diagnostic tasks 81 | -h, --help Display help/usage 82 | check_name A specific check to run 83 | ``` 84 | 85 | To get an idea of what checks will be run, use the `--list` option: 86 | 87 | ```bash 88 | riak-admin diag --list 89 | Available diagnostic checks: 90 | 91 | disk Data directory permissions and atime 92 | dumps Find crash dumps 93 | memory_use Measure memory usage 94 | nodes_connected Cluster node liveness 95 | ring_membership Cluster membership validity 96 | ring_size Ring size valid 97 | ``` 98 | 99 | If you want all the gory details about what Riaknostic is doing, you can run the checks at a more verbose logging level with the --level option: 100 | 101 | ```bash 102 | riak-admin diag --level debug 103 | 18:34:19.708 [debug] Lager installed handler lager_console_backend into lager_event 104 | 18:34:19.720 [debug] Lager installed handler error_logger_lager_h into error_logger 105 | 18:34:19.720 [info] Application lager started on node nonode@nohost 106 | 18:34:20.736 [debug] Not connected to the local Riak node, trying to connect. alive:false connect_failed:undefined 107 | 18:34:20.737 [debug] Starting distributed Erlang. 108 | 18:34:20.740 [debug] Supervisor net_sup started erl_epmd:start_link() at pid <0.42.0> 109 | 18:34:20.742 [debug] Supervisor net_sup started auth:start_link() at pid <0.43.0> 110 | 18:34:20.771 [debug] Supervisor net_sup started net_kernel:start_link(['riak_diag87813@127.0.0.1',longnames]) at pid <0.44.0> 111 | 18:34:20.771 [debug] Supervisor kernel_sup started erl_distribution:start_link(['riak_diag87813@127.0.0.1',longnames]) at pid <0.41.0> 112 | 18:34:20.781 [debug] Supervisor inet_gethost_native_sup started undefined at pid <0.49.0> 113 | 18:34:20.782 [debug] Supervisor kernel_safe_sup started inet_gethost_native:start_link() at pid <0.48.0> 114 | 18:34:20.834 [debug] Connected to local Riak node 'riak@127.0.0.1'. 115 | 18:34:20.939 [debug] Local RPC: os:getpid([]) [5000] 116 | 18:34:20.939 [debug] Running shell command: ps -o pmem,rss,command -p 83144 117 | 18:34:20.946 [debug] Shell command output: 118 | %MEM RSS COMMAND 119 | 0.4 31004 /srv/riak/erts-5.8.4/bin/beam.smp -K true -A 64 -W w -- -root /srv/riak/rel/riak -progname riak -- -home /Users/sean -- -boot /srv/riak/releases/1.0.2/riak -embedded -config /srv/riak/etc/app.config -name riak@127.0.0.1 -setcookie riak -- console 120 | 121 | 18:34:20.960 [warning] Riak crashed at Wed, 07 Dec 2011 21:47:50 GMT, leaving crash dump in /srv/riak/log/erl_crash.dump. Please inspect or remove the file. 122 | 18:34:20.961 [notice] Data directory /srv/riak/data/bitcask is not mounted with 'noatime'. Please remount its disk with the 'noatime' flag to improve performance. 123 | 18:34:20.961 [info] Riak process is using 0.4% of available RAM, totalling 31004 KB of real memory. 124 | ``` 125 | 126 | Most times you'll want to use the defaults, but any Syslog severity name will do (from most to least verbose): `debug, info, notice, warning, error, critical, alert, emergency`. 127 | 128 | Finally, if you want to run just a single diagnostic or a list of specific ones, you can pass their name(s): 129 | 130 | ```bash 131 | riak-admin diag dumps 132 | 18:41:24.083 [warning] Riak crashed at Wed, 07 Dec 2011 21:47:50 GMT, leaving crash dump in /srv/riak/log/erl_crash.dump. Please inspect or remove the file. 133 | ``` 134 | 135 | ## Contributing 136 | 137 | 0. Read DEVELOPMENT.md 138 | 1. Fork the project on [Github](https://github.com/basho/riaknostic). 139 | 2. Make your changes or additions on a "topic" branch, test and 140 | document them. If you are making a new diagnostic, make sure you 141 | give some module-level information about the checks it 142 | performs. *Note*: diagnostics _should not_ make modifications to 143 | Riak, only inspect things. 144 | 3. Push to your fork and send a pull-request. 145 | 4. A Basho Developer Advocate or Engineer will review your 146 | pull-request and get back to you. 147 | -------------------------------------------------------------------------------- /dialyzer.ignore-warnings: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/basho/riaknostic/dad8939d0ef32fbf435d13697720223293195282/dialyzer.ignore-warnings -------------------------------------------------------------------------------- /doc/overview.edoc: -------------------------------------------------------------------------------- 1 | @author Basho Technologies, Inc. 2 | @copyright 2011 Basho Technologies, Inc. 3 | @version 1.0.0 4 | @title riaknostic: Automated diagnostic tools for Riak 5 | @doc

riaknostic is an escript and set of tools that diagnoses common problems which could affect a Riak node or cluster. When experiencing any problem with Riak, riaknostic should be the first thing run during troubleshooting. The tool is integrated with Riak via the riak-admin script.

6 | 7 | 
$ riak-admin diag
8 | 9 |

This documentation describes the riaknostic API and user interface commands. The application's core consists of 5 modules:

10 | 11 | 19 | 20 |

All other included modules are generally prefixed with riaknostic_check_ and are individual diagnostics that can be run.

21 | 22 |

riaknostic is licensed under the Apache v2 license.

23 | @end -------------------------------------------------------------------------------- /priv/ForkMe_Blk.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/basho/riaknostic/dad8939d0ef32fbf435d13697720223293195282/priv/ForkMe_Blk.png -------------------------------------------------------------------------------- /priv/doctorbasho.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/basho/riaknostic/dad8939d0ef32fbf435d13697720223293195282/priv/doctorbasho.jpg -------------------------------------------------------------------------------- /priv/edoc.css: -------------------------------------------------------------------------------- 1 | /* Baseline rhythm */ 2 | body { 3 | font-size: 16px; 4 | font-family: Helvetica, sans-serif; 5 | margin: 8px; 6 | } 7 | 8 | p { 9 | font-size: 1em; /* 16px */ 10 | line-height: 1.5em; /* 24px */ 11 | margin: 0 0 1.5em 0; 12 | } 13 | 14 | h1 { 15 | font-size: 1.5em; /* 24px */ 16 | line-height: 1em; /* 24px */ 17 | margin-top: 1em; 18 | margin-bottom: 0em; 19 | } 20 | 21 | h2 { 22 | font-size: 1.375em; /* 22px */ 23 | line-height: 1.0909em; /* 24px */ 24 | margin-top: 1.0909em; 25 | margin-bottom: 0em; 26 | } 27 | 28 | h3 { 29 | font-size: 1.25em; /* 20px */ 30 | line-height: 1.2em; /* 24px */ 31 | margin-top: 1.2em; 32 | margin-bottom: 0em; 33 | } 34 | 35 | h4 { 36 | font-size: 1.125em; /* 18px */ 37 | line-height: 1.3333em; /* 24px */ 38 | margin-top: 1.3333em; 39 | margin-bottom: 0em; 40 | } 41 | 42 | .class-for-16px { 43 | font-size: 1em; /* 16px */ 44 | line-height: 1.5em; /* 24px */ 45 | margin-top: 1.5em; 46 | margin-bottom: 0em; 47 | } 48 | 49 | .class-for-14px { 50 | font-size: 0.875em; /* 14px */ 51 | line-height: 1.7143em; /* 24px */ 52 | margin-top: 1.7143em; 53 | margin-bottom: 0em; 54 | } 55 | 56 | ul { 57 | margin: 0 0 1.5em 0; 58 | } 59 | 60 | /* Customizations */ 61 | body { 62 | color: #333; 63 | } 64 | 65 | tt, code, pre { 66 | font-family: "Andale Mono", "Inconsolata", "Monaco", "DejaVu Sans Mono", monospaced; 67 | } 68 | 69 | tt, code { font-size: 0.875em } 70 | 71 | pre { 72 | font-size: 0.875em; /* 14px */ 73 | line-height: 1.7143em; /* 24px */ 74 | margin: 0 1em 1.7143em; 75 | padding: 0 1em; 76 | background: #eee; 77 | } 78 | 79 | .navbar img, hr { display: none } 80 | 81 | table { 82 | border-collapse: collapse; 83 | } 84 | 85 | h1 { 86 | border-left: 0.5em solid #fa0; 87 | padding-left: 0.5em; 88 | } 89 | 90 | h2.indextitle { 91 | font-size: 1.25em; /* 20px */ 92 | line-height: 1.2em; /* 24px */ 93 | margin: -8px -8px 0.6em; 94 | background-color: #fa0; 95 | color: white; 96 | padding: 0.3em; 97 | } 98 | 99 | ul.index { 100 | list-style: none; 101 | margin-left: 0em; 102 | padding-left: 0; 103 | } 104 | 105 | ul.index li { 106 | display: inline; 107 | padding-right: 0.75em 108 | } 109 | 110 | div.spec p { 111 | margin-bottom: 0; 112 | padding-left: 1.25em; 113 | background-color: #eee; 114 | } 115 | 116 | h3.function { 117 | border-left: 0.5em solid #fa0; 118 | padding-left: 0.5em; 119 | background: #fc9; 120 | } 121 | a, a:visited, a:hover, a:active { color: #C60 } 122 | h2 a, h3 a { color: #333 } 123 | 124 | i { 125 | font-size: 0.875em; /* 14px */ 126 | line-height: 1.7143em; /* 24px */ 127 | margin-top: 1.7143em; 128 | margin-bottom: 0em; 129 | font-style: normal; 130 | } 131 | -------------------------------------------------------------------------------- /priv/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | Riaknostic 7 | 10 | 11 | 12 | 13 |
14 |
15 |

Riaknostic 16 | diagnostic tools for Riak 17 |

18 |
19 | Download v1.0.1 » 21 |
22 |
23 |
24 |
25 |
26 |

Overview

27 |

Sometimes, things go wrong in Riak. How can you know what's 28 | wrong? Riaknostic is here to help.

29 | 
$ riak-admin diag
 30 | 15:34:52.736 [warning] Riak crashed at Wed, 07 Dec 2011 21:47:50 GMT, leaving crash dump in /srv/riak/log/erl_crash.dump. Please inspect or remove the file.
 31 | 15:34:52.736 [notice] Data directory /srv/riak/data/bitcask is not mounted with 'noatime'. Please remount its disk with the 'noatime' flag to improve performance.
32 |

Riaknostic, which is invoked via the above command, is a 33 | small suite of diagnostic checks that can be run against 34 | your Riak node to discover common problems and recommend how 35 | to resolve them. These checks are derived from the experience 36 | of the Basho Client Services Team as well as numerous 37 | public discussions on the mailing list, IRC room, and other 38 | online media.

39 |
40 |
41 |
42 |
43 |

Installation

44 |

After downloading the package, expand it in the directory below according to 45 | your platform:

46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 68 | 69 | 70 |
PlatformDirectory
Linux (Redhat, CentOS, Debian, Ubuntu)/usr/lib/riak/lib
Linux (Fedora)/usr/lib64/riak/lib
Solaris, OpenSolaris/opt/riak/lib
Mac OS/X or Self-built$RIAK/lib 66 | (where $RIAK=rel/riak for source installs, 67 | or the directory where you unpacked the package)
71 |

For example, on Linux, I might do this:

72 | 
$ wget https://github.com/basho/riaknostic/downloads/riaknostic-1.0.1.tar.gz -P /tmp
 73 | $ cd /usr/lib/riak/lib
 74 | $ sudo tar xzvf /tmp/riaknostic-1.0.1.tar.gz
75 |

The package will expand to a riaknostic/ 76 | directory which contains the riaknostic script, 77 | source code in the src/ directory and 78 | documentation. Now try it out!

79 |
80 |
81 |

Usage

82 |

For most cases, you can just run the riak-admin 83 | diag command as given at the top of the 84 | page. However, sometimes you might want to know some extra 85 | detail or run only specific checks. For that, there are 86 | command-line options. Add --help to get the options:

87 | 
$ riak-admin diag --help
 88 | Usage: riak-admin diag [-d <level>] [-l] [-h] [check_name ...]
 89 | 
 90 |   -d, --level		Minimum message severity level (default: notice)
 91 |   -l, --list		Describe available diagnostic tasks
 92 |   -h, --help		Display help/usage
 93 |   check_name		A specific check to run
94 |

To get an idea of what checks will be run, use 95 | the --list option:

96 | 
$ riak-admin diag --list
 97 | Available diagnostic checks:
 98 | 
 99 |   disk                 Data directory permissions and atime
100 |   dumps                Find crash dumps
101 |   memory_use           Measure memory usage
102 |   nodes_connected      Cluster node liveness
103 |   ring_membership      Cluster membership validity
104 |   ring_size            Ring size valid
105 |

If you want all the gory details about what Riaknostic is 106 | doing, you can run the checks at a more verbose logging 107 | level with the --level option:

108 | 
$ riak-admin diag --level debug
109 | 18:34:19.708 [debug] Lager installed handler lager_console_backend into lager_event
110 | 18:34:19.720 [debug] Lager installed handler error_logger_lager_h into error_logger
111 | 18:34:19.720 [info] Application lager started on node nonode@nohost
112 | 18:34:20.736 [debug] Not connected to the local Riak node, trying to connect. alive:false connect_failed:undefined
113 | 18:34:20.737 [debug] Starting distributed Erlang.
114 | 18:34:20.740 [debug] Supervisor net_sup started erl_epmd:start_link() at pid <0.42.0>
115 | 18:34:20.742 [debug] Supervisor net_sup started auth:start_link() at pid <0.43.0>
116 | 18:34:20.771 [debug] Supervisor net_sup started net_kernel:start_link(['riak_diag87813@127.0.0.1',longnames]) at pid <0.44.0>
117 | 18:34:20.771 [debug] Supervisor kernel_sup started erl_distribution:start_link(['riak_diag87813@127.0.0.1',longnames]) at pid <0.41.0>
118 | 18:34:20.781 [debug] Supervisor inet_gethost_native_sup started undefined at pid <0.49.0>
119 | 18:34:20.782 [debug] Supervisor kernel_safe_sup started inet_gethost_native:start_link() at pid <0.48.0>
120 | 18:34:20.834 [debug] Connected to local Riak node 'riak@127.0.0.1'.
121 | 18:34:20.939 [debug] Local RPC: os:getpid([]) [5000]
122 | 18:34:20.939 [debug] Running shell command: ps -o pmem,rss,command -p 83144
123 | 18:34:20.946 [debug] Shell command output: 
124 | %MEM    RSS COMMAND
125 |  0.4  31004 /srv/riak/erts-5.8.4/bin/beam.smp -K true -A 64 -W w -- -root /srv/riak/rel/riak -progname riak -- -home /Users/sean -- -boot /srv/riak/releases/1.0.2/riak -embedded -config /srv/riak/etc/app.config -name riak@127.0.0.1 -setcookie riak -- console
126 | 
127 | 18:34:20.960 [warning] Riak crashed at Wed, 07 Dec 2011 21:47:50 GMT, leaving crash dump in /srv/riak/log/erl_crash.dump. Please inspect or remove the file.
128 | 18:34:20.961 [notice] Data directory /srv/riak/data/bitcask is not mounted with 'noatime'. Please remount its disk with the 'noatime' flag to improve performance.
129 | 18:34:20.961 [info] Riak process is using 0.4% of available RAM, totalling 31004 KB of real memory.
130 |

Most times you'll want to use the defaults, but any 131 | Syslog severity name will do (from most to least 132 | verbose): debug, info, notice, warning, error, 133 | critical, alert, emergency.

134 |

Finally, if you want to run just a single diagnostic or a list 135 | of specific ones, you can pass their name(s):

136 | 
$ riak-admin diag dumps
137 | 18:41:24.083 [warning] Riak crashed at Wed, 07 Dec 2011 21:47:50 GMT, leaving crash dump in /srv/riak/log/erl_crash.dump. Please inspect or remove the file.
138 |
139 |
140 |
141 |

Contributing

142 |

Have an idea for a diagnostic? Want to improve the way 143 | Riaknostic works? Fork 144 | the github 145 | repository and send us a pull-request with your 146 | changes! The code is documented with edoc, 147 | so give the API Docs a 148 | read before you contribute.

149 |

If you want to run the riaknostic script 150 | while developing and you don't have it hooked up to your 151 | local Riak, you can invoke it directly like so:

152 | 
$ ./riaknostic --etc ~/code/riak/rel/riak/etc --base ~/code/riak/rel/riak --user `whoami` [other options]
153 |

Those extra options are usually assigned by 154 | the riak-admin script for you, but here's 155 | how to set them:

156 | 157 | 158 | 159 | 162 | 163 | 164 | 165 | 171 | 172 | 173 | 174 | 177 | 178 |
--etcWhere your Riak configuration directory is, in the 160 | example above it's in the generated directory of a 161 | source checkout of Riak.
--baseThe "base" directory of Riak, usually the root of 166 | the generated directory 167 | or /usr/lib/riak on Linux, for 168 | example. Scan the riak-admin script for 169 | how the RUNNER_BASE_DIR variable is 170 | assigned on your platform.
--userWhat user/UID the Riak node runs as. In a source 175 | checkout, it's the current user, on most systems, 176 | it's riak.
179 |
180 |
181 |
182 | 183 | 184 | -------------------------------------------------------------------------------- /rebar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/basho/riaknostic/dad8939d0ef32fbf435d13697720223293195282/rebar -------------------------------------------------------------------------------- /rebar.config: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riaknostic - automated diagnostic tools for Riak 4 | %% 5 | %% Copyright (c) 2011 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | {escript_shebang, "#!/usr/bin/env escript\n"}. 24 | {escript_comment, "%% -nocookie\n"}. 25 | 26 | {erl_opts, [debug_info, {parse_transform, lager_transform}]}. 27 | 28 | {escript_incl_apps, [lager, getopt, goldrush]}. 29 | 30 | {deps, [ 31 | {lager, ".*", {git, "https://github.com/basho/lager.git", {tag, "3.2.4"}}}, 32 | {getopt, ".*", {git, "https://github.com/basho/getopt.git", {tag, "v0.8.2"}}}, 33 | {meck, "0.8.*", {git, "https://github.com/basho/meck.git", {tag, "0.8.2"}}} 34 | ]}. 35 | 36 | {edoc_opts, [{stylesheet_file, "priv/edoc.css"}]}. 37 | -------------------------------------------------------------------------------- /src/riaknostic.app.src: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riaknostic - automated diagnostic tools for Riak 4 | %% 5 | %% Copyright (c) 2011 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | {application, riaknostic, 23 | [ 24 | {description, "Diagnostic tools for Riak"}, 25 | {vsn, git}, 26 | {registered, []}, 27 | {applications, [ 28 | kernel, 29 | stdlib, 30 | inets, 31 | lager 32 | ]}, 33 | {mod, { riaknostic_app, []}} 34 | ]}. 35 | -------------------------------------------------------------------------------- /src/riaknostic.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riaknostic - automated diagnostic tools for Riak 4 | %% 5 | %% Copyright (c) 2011 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | %% @doc

The riaknostic module is the entry point for 24 | %% the escript. It is responsible for parsing command-line arguments 25 | %% and switches, printing the available checks, listing the help text, 26 | %% or running all or the specified checks, depending on the command 27 | %% line.

28 | %% 29 | %%

The getopt application and module is used 30 | %% for command-line parsing. The defined switches and arguments are:

31 | %% 
$ ./riaknostic --etc etc --base base --user user [-d level] [-l] [-h] [check_name...]
32 | %% 33 | %% 34 | %% 37 | %% 40 | %% 42 | %% 45 | %% 48 | %% 50 | %% 52 | %%
--etc etcthe location of the Riak 35 | %% configuration directory (set automatically by 36 | %% riak-admin)
--base basethe base directory of 38 | %% Riak, aka RUNNER_BASE_DIR (set automatically by 39 | %% riak-admin)
--user userthe user that Riak runs as 41 | %% (set automatically by riak-admin)
-d, --level level  the severity of 43 | %% messages you want to see, defaulting to 'notice'. Equivalent to 44 | %% syslog/lager severity levels.
-l, --listlists available checks, 46 | %% that is, modules that implement riaknostic_check. A 47 | %% "short name" will be given for ease-of-use.
-h, --help - print command usage 49 | %% ("help")
check_namewhen given, a specific 51 | %% check or list of checks to run
53 | %% @end 54 | -module(riaknostic). 55 | -export([main/1]). 56 | 57 | -include_lib("lager/include/lager.hrl"). 58 | 59 | -define(OPTS, [ 60 | {etc, undefined, "etc", string, undefined }, 61 | {base, undefined, "base", string, undefined }, 62 | {user, undefined, "user", string, undefined }, 63 | {level, $d, "level", {atom, notice}, "Minimum message severity level (default: notice)"}, 64 | {list, $l, "list", undefined, "Describe available diagnostic tasks" } 65 | ]). 66 | 67 | %% @doc The main entry point for the riaknostic escript. 68 | -spec main(CommandLineArguments::[string()]) -> any(). 69 | main(Args) -> 70 | application:load(riaknostic), 71 | 72 | case getopt:parse(?OPTS, Args) of 73 | {ok, {Opts, NonOptArgs}} -> 74 | case process_opts(Opts) of 75 | list -> list_checks(); 76 | run -> run(NonOptArgs) 77 | end; 78 | {error, Error} -> 79 | io:format("Invalid option sequence given: ~w~n", [Error]) 80 | end. 81 | 82 | list_checks() -> 83 | Descriptions = [ {riaknostic_util:short_name(Mod), Mod:description()} || 84 | Mod <- riaknostic_check:modules() ], 85 | io:format("Available diagnostic checks:~n~n"), 86 | lists:foreach(fun({Mod, Desc}) -> 87 | io:format(" ~.20s ~s~n", [Mod, Desc]) 88 | end, lists:sort(Descriptions)). 89 | 90 | run(InputChecks) -> 91 | case riaknostic_config:prepare() of 92 | {error, Reason} -> 93 | io:format("Fatal error: ~s~n", [Reason]); 94 | _ -> 95 | ok 96 | end, 97 | Checks = case InputChecks of 98 | [] -> 99 | riaknostic_check:modules(); 100 | _ -> 101 | ShortNames = [{riaknostic_util:short_name(Mod), Mod} || Mod <- riaknostic_check:modules() ], 102 | element(1, lists:foldr(fun validate_checks/2, {[], ShortNames}, InputChecks)) 103 | end, 104 | Messages = lists:foldl(fun(Mod, Acc) -> 105 | Acc ++ riaknostic_check:check(Mod) 106 | end, [], Checks), 107 | case Messages of 108 | [] -> 109 | io:format("No diagnostic messages to report.~n"); 110 | _ -> 111 | %% Print the most critical messages first 112 | LogLevelNum = lists:foldl( 113 | fun({mask, Mask}, Acc) -> 114 | Mask bor Acc; 115 | (Level, Acc) when is_integer(Level) -> 116 | {mask, Mask} = lager_util:config_to_mask(lager_util:num_to_level(Level)), 117 | Mask bor Acc; 118 | (_, Acc) -> 119 | Acc 120 | end, 0, lager:get_loglevels(?DEFAULT_SINK)), 121 | FilteredMessages = lists:filter(fun({Level,_,_}) -> 122 | lager_util:level_to_num(Level) =< LogLevelNum 123 | end, Messages), 124 | SortedMessages = lists:sort(fun({ALevel, _, _}, {BLevel, _, _}) -> 125 | lager_util:level_to_num(ALevel) =< lager_util:level_to_num(BLevel) 126 | end, FilteredMessages), 127 | case SortedMessages of 128 | [] -> 129 | io:format("No diagnostic messages to report.~n"); 130 | _ -> 131 | lists:foreach(fun riaknostic_check:print/1, SortedMessages) 132 | end 133 | end. 134 | 135 | validate_checks(Check, {Mods, SNames}) -> 136 | case lists:keyfind(Check, 1, SNames) of 137 | {Check, Mod} -> 138 | {[Mod|Mods], lists:delete({Check, Mod}, SNames)}; 139 | _ -> 140 | io:format("Unknown check '~s' specified, skipping.~n", [Check]), 141 | {Mods, SNames} 142 | end. 143 | 144 | process_opts(Opts) -> 145 | process_opts(Opts, run). 146 | 147 | process_opts([], Result) -> 148 | Result; 149 | process_opts([H|T], Result) -> 150 | process_opts(T, process_option(H, Result)). 151 | 152 | process_option({etc,Path}, Result) -> 153 | application:set_env(riaknostic, etc, filename:absname(Path)), 154 | Result; 155 | process_option({base, Path}, Result) -> 156 | application:set_env(riaknostic, base, filename:absname(Path)), 157 | Result; 158 | process_option({user, User}, Result) -> 159 | application:set_env(riaknostic, user, User), 160 | Result; 161 | process_option({level, Level}, Result) -> 162 | application:set_env(riaknostic, log_level, Level), 163 | Result; 164 | process_option(list, usage) -> %% Help should have precedence over listing checks 165 | usage; 166 | process_option(list, _) -> 167 | list; 168 | process_option(usage, _) -> 169 | usage. 170 | -------------------------------------------------------------------------------- /src/riaknostic_check.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riaknostic - automated diagnostic tools for Riak 4 | %% 5 | %% Copyright (c) 2011 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | %% @doc

Enforces a common API among all diagnostic modules and 24 | %% provides some automation around their execution.

25 | %%

Behaviour Specification

26 | %% 27 | %%

description/0

28 | %% 
-spec description() -> iodata().
29 | %%

A short description of what the diagnostic does, which will be 30 | %% printed when the script is given the -l flag.

31 | %% 32 | %%

valid/0

33 | %% 
-spec valid() -> boolean().
34 | %%

Whether the diagnostic is valid to run. For example, some checks 35 | %% require connectivity to the Riak node and hence call {@link 36 | %% riaknostic_node:can_connect/0. riaknostic_node:can_connect()}.

37 | %% 38 | %%

check/0

39 | %% 
-spec check() -> [{lager:log_level(), term()}].
40 | %%

Runs the diagnostic, returning a list of pairs, where the first 41 | %% is a severity level and the second is any term that is understood 42 | %% by the format/1 callback.

43 | %% 44 | %%

format/1

45 | %% 
-spec format(term()) -> iodata() | {io:format(), [term()]}.
46 | %%

Formats terms that were returned from check/0 for 47 | %% output to the console. Valid return values are an iolist (string, 48 | %% binary, etc) or a pair of a format string and a list of terms, as 49 | %% you would pass to {@link io:format/2. io:format/2}.

50 | %% @end 51 | 52 | -module(riaknostic_check). 53 | -export([behaviour_info/1]). 54 | -export([check/1, 55 | modules/0, 56 | print/1]). 57 | 58 | %% @doc The behaviour definition for diagnostic modules. 59 | -spec behaviour_info(atom()) -> 'undefined' | [{atom(), arity()}]. 60 | behaviour_info(callbacks) -> 61 | [{description, 0}, 62 | {valid, 0}, 63 | {check, 0}, 64 | {format, 1}]; 65 | behaviour_info(_) -> 66 | undefined. 67 | 68 | %% @doc Runs the diagnostic in the given module, if it is valid. Returns a 69 | %% list of messages that will be printed later using print/1. 70 | -spec check(Module::module()) -> [{lager:log_level(), module(), term()}]. 71 | check(Module) -> 72 | case Module:valid() of 73 | true -> 74 | [ {Level, Module, Message} || {Level, Message} <- Module:check() ]; 75 | _ -> 76 | [] 77 | end. 78 | 79 | %% @doc Collects a list of diagnostic modules included in the 80 | %% riaknostic application. 81 | -spec modules() -> [module()]. 82 | modules() -> 83 | {ok, Mods} = application:get_key(riaknostic, modules), 84 | [ M || M <- Mods, 85 | Attr <- M:module_info(attributes), 86 | {behaviour, [?MODULE]} =:= Attr orelse {behavior, [?MODULE]} =:= Attr ]. 87 | 88 | 89 | %% @doc Formats and prints the given message via lager:log/3,4. The diagnostic 90 | %% module's format/1 function will be called to provide a 91 | %% human-readable message. It should return an iolist() or a 2-tuple 92 | %% consisting of a format string and a list of terms. 93 | -spec print({Level::lager:log_level(), Module::module(), Data::term()}) -> ok. 94 | print({Level, Mod, Data}) -> 95 | case Mod:format(Data) of 96 | {Format, Terms} -> 97 | riaknostic_util:log(Level, Format, Terms); 98 | String -> 99 | riaknostic_util:log(Level, String) 100 | end. 101 | 102 | 103 | -------------------------------------------------------------------------------- /src/riaknostic_check_disk.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riaknostic - automated diagnostic tools for Riak 4 | %% 5 | %% Copyright (c) 2011 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | %% @doc Diagnostic that checks permissions on data directories and 24 | %% whether noatime is set. It will only check data directories of 25 | %% known storage backends. 26 | -module(riaknostic_check_disk). 27 | -behaviour(riaknostic_check). 28 | 29 | %% The file that we will attempt to create and read under each data directory. 30 | -define(TEST_FILE, "riaknostic.tmp"). 31 | 32 | %% A dependent chain of permissions checking functions. 33 | -define(CHECKPERMFUNS, [fun check_is_dir/1, 34 | fun check_is_writeable/1, 35 | fun check_is_readable/1, 36 | fun check_is_file_readable/1, 37 | fun check_atime/1]). 38 | 39 | -include_lib("kernel/include/file.hrl"). 40 | 41 | -export([description/0, 42 | valid/0, 43 | check/0, 44 | format/1]). 45 | 46 | -spec description() -> string(). 47 | description() -> 48 | "Data directory permissions and atime". 49 | 50 | -spec valid() -> true. 51 | valid() -> 52 | true. 53 | 54 | -spec check() -> [{lager:log_level(), term()}]. 55 | check() -> 56 | DataDirs = riaknostic_config:data_directories(), 57 | %% Add additional disk checks in the function below 58 | lists:flatmap(fun(Dir) -> 59 | check_directory_permissions(Dir) 60 | end, 61 | DataDirs). 62 | 63 | -spec format(term()) -> {io:format(), [term()]}. 64 | format({disk_full, DataDir}) -> 65 | {"Disk containing data directory ~s is full! " 66 | "Please check that it is set to the correct location and that there are not " 67 | "other files using up space intended for Riak.", [DataDir]}; 68 | format({no_data_dir, DataDir}) -> 69 | {"Data directory ~s does not exist. Please create it.", [DataDir]}; 70 | format({no_write, DataDir}) -> 71 | User = riaknostic_config:user(), 72 | {"No write access to data directory ~s. Please make it writeable by the '~s' user.", [DataDir, User]}; 73 | format({no_read, DataDir}) -> 74 | User = riaknostic_config:user(), 75 | {"No read access to data directory ~s. Please make it readable by the '~s' user.", [DataDir, User]}; 76 | format({write_check, File}) -> 77 | {"Write-test file ~s is a directory! Please remove it so this test can continue.", [File]}; 78 | format({atime, Dir}) -> 79 | {"Data directory ~s is not mounted with 'noatime'. " 80 | "Please remount its disk with the 'noatime' flag to improve performance.", [Dir]}. 81 | 82 | %%% Private functions 83 | 84 | check_directory_permissions(Directory) -> 85 | check_directory(Directory, ?CHECKPERMFUNS). 86 | 87 | %% Run a list of check functions against the given directory, 88 | %% returning the first non-ok result. 89 | check_directory(_, []) -> 90 | []; 91 | check_directory(Directory, [Check|Checks]) -> 92 | case Check(Directory) of 93 | ok -> 94 | check_directory(Directory, Checks); 95 | Message -> 96 | [ Message ] 97 | end. 98 | 99 | %% Check if the path is actually a directory 100 | check_is_dir(Directory) -> 101 | case filelib:is_dir(Directory) of 102 | true -> 103 | ok; 104 | _ -> 105 | {error, {no_data_dir, Directory}} 106 | end. 107 | 108 | %% Check if the directory is writeable 109 | check_is_writeable(Directory) -> 110 | File = filename:join([Directory, ?TEST_FILE]), 111 | case file:write_file(File, <<"ok">>) of 112 | ok -> 113 | ok; 114 | {error, Error} when Error == enoent orelse Error == eacces -> 115 | {error, {no_write, Directory}}; 116 | {error, enospc} -> 117 | {critical, {disk_full, Directory}}; 118 | {error, eisdir} -> 119 | {error, {write_check, File}} 120 | end. 121 | 122 | %% Check if the directory is readable 123 | check_is_readable(Directory) -> 124 | case file:read_file_info(Directory) of 125 | {ok, #file_info{access=Access}} when Access == read orelse 126 | Access == read_write -> 127 | ok; 128 | {error, eacces} -> 129 | {error, {no_read, Directory}}; 130 | {error, Error} when Error == enoent orelse 131 | Error == enotdir -> 132 | {error, {no_data_dir, Directory}}; 133 | _ -> 134 | {error, {no_read, Directory}} 135 | end. 136 | 137 | %% Check if the file we created is readable 138 | check_is_file_readable(Directory) -> 139 | File = filename:join([Directory, ?TEST_FILE]), 140 | case file:read_file(File) of 141 | {error, Error} when Error == eacces orelse 142 | Error == enotdir -> 143 | {error, {no_read, Directory}}; 144 | {error, enoent} -> 145 | {error, {write_check, File}}; 146 | _ -> ok 147 | end. 148 | 149 | %% Check if the directory is mounted with 'noatime' 150 | check_atime(Directory) -> 151 | File = filename:join([Directory, ?TEST_FILE]), 152 | {ok, FileInfo1} = file:read_file_info(File), 153 | timer:sleep(1001), 154 | {ok, S} = file:open(File, [read]), 155 | io:get_line(S, ''), 156 | file:close(S), 157 | {ok, FileInfo2} = file:read_file_info(File), 158 | file:delete(File), 159 | case (FileInfo1#file_info.atime =/= FileInfo2#file_info.atime) of 160 | true -> 161 | {notice, {atime, Directory}}; 162 | _ -> 163 | ok 164 | end. 165 | 166 | -------------------------------------------------------------------------------- /src/riaknostic_check_dumps.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riaknostic - automated diagnostic tools for Riak 4 | %% 5 | %% Copyright (c) 2011 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | %% @doc Diagnostic that detects the existence of Erlang-generated 24 | %% crash dumps. It will also check whether the location that the crash 25 | %% dump is written to has correct permissions. 26 | -module(riaknostic_check_dumps). 27 | -behaviour(riaknostic_check). 28 | 29 | -include_lib("kernel/include/file.hrl"). 30 | 31 | -export([description/0, 32 | valid/0, 33 | check/0, 34 | format/1]). 35 | 36 | -spec description() -> string(). 37 | description() -> 38 | "Find crash dumps". 39 | 40 | -spec valid() -> true. 41 | valid() -> 42 | true. 43 | 44 | -spec check() -> [{lager:log_level(), term()}]. 45 | check() -> 46 | CrashDumpConfig = riaknostic_config:get_vm_env("ERL_CRASH_DUMP"), 47 | {DumpDir, DumpFile} = case CrashDumpConfig of 48 | undefined -> 49 | Cwd = riaknostic_config:base_dir(), 50 | {Cwd, filename:absname([Cwd, "erl_crash.dump"])}; 51 | File -> 52 | AbsFile = filename:absname(File, riaknostic_config:base_dir()), 53 | {filename:dirname(AbsFile), AbsFile} 54 | end, 55 | Messages = case file:read_file_info(DumpDir) of 56 | {error, enoent} -> 57 | [{error, {enoent, DumpDir}}]; 58 | {error, _} -> 59 | [{error, {eacces, DumpDir}}]; 60 | {ok, #file_info{access=Access}} when Access =/= read_write -> 61 | [{error, {eacces, DumpDir}}]; 62 | _ -> 63 | [] 64 | end, 65 | case filelib:is_file(DumpFile) of 66 | true -> 67 | [{warning, {crash_dump, DumpFile}}|Messages]; 68 | _ -> 69 | Messages 70 | end. 71 | 72 | -spec format(term()) -> {io:format(), [term()]}. 73 | format({eacces, Dir}) -> 74 | {"Crash dump directory ~s is not writeable by Riak. Please set -env ERL_CRASH_DUMP /erl_crash.dump in vm.args to a writeable path.", [Dir]}; 75 | format({enoent, Dir}) -> 76 | {"Crash dump directory ~s does not exist. Please set -env ERL_CRASH_DUMP /erl_crash.dump in vm.args to a writeable path.", [Dir]}; 77 | format({crash_dump, File}) -> 78 | {ok, #file_info{mtime=MTime}} = file:read_file_info(File), 79 | {"Riak crashed at ~s, leaving crash dump in ~s. Please inspect or remove the file.", [httpd_util:rfc1123_date(MTime), File]}. 80 | -------------------------------------------------------------------------------- /src/riaknostic_check_memory_use.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riaknostic - automated diagnostic tools for Riak 4 | %% 5 | %% Copyright (c) 2011 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | %% @doc Diagnostic that checks Riak's current memory usage. If memory 24 | %% usage is high, a warning message will be sent, otherwise only 25 | %% informational messages. 26 | -module(riaknostic_check_memory_use). 27 | -behaviour(riaknostic_check). 28 | 29 | -export([description/0, 30 | valid/0, 31 | check/0, 32 | format/1]). 33 | 34 | -spec description() -> string(). 35 | description() -> 36 | "Measure memory usage". 37 | 38 | -spec valid() -> boolean(). 39 | valid() -> 40 | riaknostic_node:can_connect(). 41 | 42 | -spec check() -> [{lager:log_level(), term()}]. 43 | check() -> 44 | Pid = riaknostic_node:pid(), 45 | Output = riaknostic_util:run_command("ps -o pmem,rss -p " ++ Pid), 46 | [_,_,Percent, RealSize| _] = string:tokens(Output, "/n \n"), 47 | Messages = [ 48 | {info, {process_usage, Percent, RealSize}} 49 | ], 50 | case riaknostic_util:binary_to_float(list_to_binary(Percent)) >= 90 of 51 | false -> 52 | Messages; 53 | true -> 54 | [{critical, {high_memory, Percent}} | Messages] 55 | end. 56 | 57 | -spec format(term()) -> {io:format(), [term()]}. 58 | format({high_memory, Percent}) -> 59 | {"Riak memory usage is HIGH: ~s% of available RAM", [Percent]}; 60 | format({process_usage, Percent, Real}) -> 61 | {"Riak process is using ~s% of available RAM, totalling ~s KB of real memory.", [Percent, Real]}. 62 | -------------------------------------------------------------------------------- /src/riaknostic_check_monitors.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riaknostic - automated diagnostic tools for Riak 4 | %% 5 | %% Copyright (c) 2011 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | %% @doc Diagnostic that notes processes which have a high monitor count(>50) 24 | -module(riaknostic_check_monitors). 25 | -behaviour(riaknostic_check). 26 | 27 | -export([description/0, 28 | valid/0, 29 | check/0, 30 | format/1]). 31 | 32 | -spec description() -> string(). 33 | description() -> 34 | "Note processes with >50 monitors". 35 | 36 | -spec valid() -> boolean(). 37 | valid() -> 38 | riaknostic_node:can_connect(). 39 | 40 | -spec check() -> [{lager:log_level(), term()}]. 41 | check() -> 42 | Fun = fun() -> [{Pid,NumMon} || Pid <- processes(), 43 | [{monitors,Mon}] <- [process_info(Pid, [monitors])], 44 | NumMon <- [length(Mon)], 45 | NumMon > 50] end, 46 | case riaknostic_node:local_command(erlang, apply, [Fun,[]]) of 47 | [] -> 48 | []; 49 | Pids -> 50 | [{warning, {high_monitor_count, Pids}}] 51 | end. 52 | 53 | -spec format(term()) -> {io:format(), [term()]}. 54 | format({high_monitor_count, Pids}) -> 55 | {"The following processes have more than 50 monitors: ~p", [Pids]}. -------------------------------------------------------------------------------- /src/riaknostic_check_nodes_connected.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riaknostic - automated diagnostic tools for Riak 4 | %% 5 | %% Copyright (c) 2011 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | %% @doc Diagnostic check that detects cluster members that are down. 24 | -module(riaknostic_check_nodes_connected). 25 | -behaviour(riaknostic_check). 26 | 27 | -export([description/0, 28 | valid/0, 29 | check/0, 30 | format/1]). 31 | 32 | -spec description() -> string(). 33 | description() -> 34 | "Cluster node liveness". 35 | 36 | -spec valid() -> boolean(). 37 | valid() -> 38 | riaknostic_node:can_connect(). 39 | 40 | -spec check() -> [{lager:log_level(), term()}]. 41 | check() -> 42 | Stats = riaknostic_node:stats(), 43 | {connected_nodes, ConnectedNodes} = lists:keyfind(connected_nodes, 1, Stats), 44 | {ring_members, RingMembers} = lists:keyfind(ring_members, 1, Stats), 45 | {nodename, NodeName} = lists:keyfind(nodename, 1, Stats), 46 | 47 | [ {warning, {node_disconnected, N}} || N <- RingMembers, 48 | N =/= NodeName, 49 | lists:member(N, ConnectedNodes) == false]. 50 | 51 | -spec format(term()) -> {io:format(), [term()]}. 52 | format({node_disconnected, Node}) -> 53 | {"Cluster member ~s is not connected to this node. Please check whether it is down.", [Node]}. 54 | -------------------------------------------------------------------------------- /src/riaknostic_check_ring_membership.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riaknostic - automated diagnostic tools for Riak 4 | %% 5 | %% Copyright (c) 2011 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | %% @doc Diagnostic that checks whether the local node is a member of 24 | %% the ring. This might arise when the node name in vm.args has 25 | %% changed but the node has not been renamed in the ring. 26 | -module(riaknostic_check_ring_membership). 27 | -behaviour(riaknostic_check). 28 | 29 | -export([description/0, 30 | valid/0, 31 | check/0, 32 | format/1]). 33 | 34 | -include_lib("eunit/include/eunit.hrl"). 35 | 36 | -spec description() -> string(). 37 | description() -> 38 | "Cluster membership validity". 39 | 40 | -spec valid() -> boolean(). 41 | valid() -> 42 | riaknostic_node:can_connect(). 43 | 44 | -spec check() -> [{lager:log_level(), term()}]. 45 | check() -> 46 | Stats = riaknostic_node:stats(), 47 | {ring_members, RingMembers} = lists:keyfind(ring_members, 1, Stats), 48 | {nodename, NodeName} = lists:keyfind(nodename, 1, Stats), 49 | case lists:member(NodeName, RingMembers) of 50 | true -> 51 | []; 52 | false -> 53 | [{warning, {not_ring_member, NodeName}}] 54 | end. 55 | 56 | check_test() -> 57 | meck:new(riaknostic_node, [passthrough]), 58 | meck:expect(riaknostic_node, stats, fun() -> [{ring_members, ["riak@127.0.0.1"]}, {nodename, ["notmember@127.0.0.1"]}] end), 59 | ?assert(meck:validate(riaknostic_node)), 60 | ?assertEqual([{warning, {not_ring_member, ["notmember@127.0.0.1"]}}], check()), 61 | ?assertNotEqual([{warning, {not_ring_member, ["notequal@127.0.0.1"]}}], check()), 62 | meck:unload(riaknostic_node). 63 | 64 | -spec format(term()) -> {io:format(), [term()]}. 65 | format({not_ring_member, Nodename}) -> 66 | {"Local node ~w is not a member of the ring. Please check that the -name setting in vm.args is correct.", [Nodename]}. 67 | -------------------------------------------------------------------------------- /src/riaknostic_check_ring_preflists.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riaknostic - automated diagnostic tools for Riak 4 | %% 5 | %% Copyright (c) 2011 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | %% @doc Diagnostic that checks the local ring for any preflists that do 24 | %% not satisfy n_val 25 | -module(riaknostic_check_ring_preflists). 26 | -behaviour(riaknostic_check). 27 | 28 | -export([description/0, 29 | valid/0, 30 | check/0, 31 | format/1]). 32 | 33 | -spec description() -> string(). 34 | description() -> 35 | "Check ring satisfies n_val". 36 | 37 | -spec valid() -> boolean(). 38 | valid() -> 39 | riaknostic_node:can_connect(). 40 | 41 | -spec check() -> [{lager:log_level(), term()}]. 42 | check() -> 43 | case riaknostic_node:local_command(riak_core_ring_util, check_ring) of 44 | [] -> []; 45 | PrefLists -> [ {warning, {n_val_not_satisfied, PrefLists}} ] 46 | end. 47 | 48 | -spec format(term()) -> {io:format(), [term()]}. 49 | format({n_val_not_satisfied, PrefLists}) -> 50 | {"The following preflists do not satisfy the n_val. Please add more nodes. ~p", [PrefLists]}. 51 | 52 | -------------------------------------------------------------------------------- /src/riaknostic_check_ring_size.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riaknostic - automated diagnostic tools for Riak 4 | %% 5 | %% Copyright (c) 2011 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | %% @doc Diagnostic that compares the configured 24 | %% ring_creation_size to the actual size of the ring. 25 | -module(riaknostic_check_ring_size). 26 | -behaviour(riaknostic_check). 27 | 28 | -export([description/0, 29 | valid/0, 30 | check/0, 31 | format/1]). 32 | 33 | -spec description() -> string(). 34 | description() -> 35 | "Ring size valid". 36 | 37 | -spec valid() -> boolean(). 38 | valid() -> 39 | riaknostic_node:can_connect(). 40 | 41 | -spec check() -> [{lager:log_level(), term()}]. 42 | check() -> 43 | Stats = riaknostic_node:stats(), 44 | {ring_creation_size, RingSize} = lists:keyfind(ring_creation_size, 1, Stats), 45 | {ring_num_partitions, NumPartitions} = lists:keyfind(ring_num_partitions, 1, Stats), 46 | % {ring_members, RingMembers} = lists:keyfind(ring_members, 1, Stats), 47 | % NumRingMembers = length(RingMembers), 48 | % VnodesPerNode = erlang:round(RingSize / NumRingMembers), 49 | % MinAcceptableVnodesPerNode = erlang:round(RingSize * 0.03), 50 | % MaxRecommendedVnodesPerNode = erlang:round(RingSize * 0.7), 51 | 52 | lists:append([ 53 | [ {notice, {ring_size_unequal, RingSize, NumPartitions}} || RingSize /= NumPartitions ], 54 | [ {critical, {ring_size_not_exp2, RingSize}} || (RingSize band -(bnot RingSize)) /= RingSize] 55 | % [ {notice, {ring_size_too_small, RingSize, NumRingMembers}} || VnodesPerNode =< MinAcceptableVnodesPerNode ], 56 | % [ {notice, {too_few_nodes_for_ring, RingSize, NumRingMembers}} || VnodesPerNode >= MaxRecommendedVnodesPerNode ] 57 | ]). 58 | 59 | -spec format(term()) -> {io:format(), [term()]}. 60 | format({ring_size_unequal, S, P}) -> 61 | {"The configured ring_creation_size (~B) is not equal to the number of partitions in the ring (~B). " 62 | "Please verify that the ring_creation_size in app.config is correct.", [S, P]}; 63 | 64 | format({ring_size_not_exp2, S}) -> 65 | {"The configured ring_creation_size (~B) should always be a power of 2. " 66 | "Please reconfigure the ring_creation_size in app.config.", [S]}. 67 | 68 | %format({ring_size_too_small, S, N}) -> 69 | % {"With a ring_creation_size (~B) and ~B nodes participating in the cluster, each node is responsible for less than 3% of the data. " 70 | % " You have too many nodes for this size ring. " 71 | % "Please consider migrating data to a cluster with 2 or 4x your current ring size.", [S, N]}; 72 | 73 | %format({too_few_nodes_for_ring, S, N}) -> 74 | % {"With a ring_creation_size (~B) and ~B nodes participating in the cluster, each node is responsible for more than 70% of the data. " 75 | % " You have too few nodes for this size ring. " 76 | % "Please consider joining more nodes to your cluster.", [S, N]}. 77 | -------------------------------------------------------------------------------- /src/riaknostic_check_search.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riaknostic - automated diagnostic tools for Riak 4 | %% 5 | %% Copyright (c) 2011 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | %% @doc Diagnostic that checks if riak_search 24 | %% is enabled on every node 25 | -module(riaknostic_check_search). 26 | -behaviour(riaknostic_check). 27 | 28 | -export([description/0, 29 | valid/0, 30 | check/0, 31 | format/1]). 32 | 33 | -spec description() -> string(). 34 | description() -> 35 | "Check whether search is enabled on all nodes". 36 | 37 | -spec valid() -> boolean(). 38 | valid() -> 39 | riaknostic_node:can_connect_all(). 40 | 41 | -spec check() -> [{lager:log_level(), term()}]. 42 | check() -> 43 | Stats = riaknostic_node:stats(), 44 | {ring_members, RingMembers} = lists:keyfind(ring_members, 1, Stats), 45 | 46 | {SearchEnabled, _} = riaknostic_node:cluster_command(application, get_env, [riak_search, enabled]), 47 | 48 | {_, X} = lists:unzip(SearchEnabled), 49 | NodesSearchEnabled = lists:zip(RingMembers, X), 50 | 51 | lists:append([ 52 | [ {warning, {riak_search, NodesSearchEnabled}} || length(lists:usort(SearchEnabled)) > 1 ] 53 | ]). 54 | 55 | -spec format(term()) -> {io:format(), [term()]}. 56 | format({riak_search, Services}) -> 57 | {"Search is not enabled on all nodes: ~p", [Services]}. 58 | -------------------------------------------------------------------------------- /src/riaknostic_check_strong_consistency.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riaknostic - automated diagnostic tools for Riak 4 | %% 5 | %% Copyright (c) 2011 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | %% @doc Diagnostic that checks Riak's current memory usage. If memory 24 | %% usage is high, a warning message will be sent, otherwise only 25 | %% informational messages. 26 | -module(riaknostic_check_strong_consistency). 27 | -behaviour(riaknostic_check). 28 | 29 | -export([description/0, 30 | valid/0, 31 | check/0, 32 | format/1]). 33 | 34 | -spec description() -> string(). 35 | description() -> 36 | "Strong consistency configuration valid". 37 | 38 | -spec valid() -> boolean(). 39 | valid() -> 40 | riaknostic_node:can_connect(). 41 | 42 | -spec check() -> [{lager:log_level(), term()}]. 43 | check() -> 44 | StrongConsistencyOption = riaknostic_config:get_app_env([riak_core, enable_consensus]), 45 | { AAEOption, _ } = riaknostic_config:get_app_env([riak_kv, anti_entropy]), 46 | maybe_strong_consistency_aae_misconfigured(StrongConsistencyOption, AAEOption). 47 | 48 | -spec maybe_strong_consistency_aae_misconfigured(boolean, on | off | any()) -> [ { term(), term() } ] | []. 49 | maybe_strong_consistency_aae_misconfigured(true, off) -> 50 | [ { critical, { strong_consistency_aae_misconfigured } } ]; 51 | maybe_strong_consistency_aae_misconfigured(false, _) -> 52 | []; 53 | maybe_strong_consistency_aae_misconfigured(true, on) -> 54 | []. 55 | 56 | -spec format(term()) -> {io:format(), [term()]}. 57 | format({ strong_consistency_aae_misconfigured }) -> 58 | { "Strong consistency has been enabled without AAE -- all consistent operations will timeout until AAE is enabled.", [] }. 59 | -------------------------------------------------------------------------------- /src/riaknostic_config.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riaknostic - automated diagnostic tools for Riak 4 | %% 5 | %% Copyright (c) 2011 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | %% @doc Provides convenient access to Riak configuration values. When 24 | %% the {@link riaknostic. riaknostic} module calls {@link 25 | %% prepare/0. prepare/0}, Riak's app.config and 26 | %% vm.args files will be parsed and memoized, and lager 27 | %% will be started on the console at the configured severity level. 28 | %% @end 29 | 30 | -module(riaknostic_config). 31 | 32 | -export([prepare/0, 33 | data_directories/0, 34 | get_app_env/1, 35 | get_app_env/2, 36 | get_vm_env/1, 37 | base_dir/0, 38 | etc_dir/0, 39 | node_name/0, 40 | cookie/0, 41 | user/0]). 42 | 43 | %% @doc Prepares appropriate configuration so the riaknostic script 44 | %% can run. This is called by the riaknostic module and you do 45 | %% not need to invoke it. 46 | -spec prepare() -> ok | {error, iodata()}. 47 | prepare() -> 48 | prepare([fun start_lager/0, fun load_app_config/0, fun load_vm_args/0]). 49 | 50 | prepare([]) -> 51 | ok; 52 | prepare([Fun|T]) -> 53 | case Fun() of 54 | {error, Reason} -> 55 | {error, Reason}; 56 | _ -> 57 | prepare(T) 58 | end. 59 | 60 | %% @doc Determines where Riak is configured to store data. Returns a 61 | %% list of paths to directories defined by storage backends. 62 | -spec data_directories() -> [ file:filename() ]. 63 | data_directories() -> 64 | KVBackend = get_app_env([riak_kv, storage_backend]), 65 | SearchBackend = get_app_env([riak_search, storage_backend], merge_index_backend), 66 | Dirs = case get_app_env([riak_search, enabled]) of 67 | true -> 68 | data_directory(KVBackend) ++ data_directory(SearchBackend); 69 | _ -> 70 | data_directory(KVBackend) 71 | end, 72 | [ filename:absname(Dir, base_dir()) || Dir <- Dirs, Dir =/= undefined ]. 73 | 74 | %% @doc Get a key out of the app.config file, or if it doesn't exist, 75 | %% return the Default. You specify a nested key using a list of atoms, 76 | %% e.g. [riak_kv, storage_backend]. 77 | %% @see get_app_env/1 78 | -spec get_app_env([atom()], term()) -> term(). 79 | get_app_env(Keys, Default) -> 80 | case get_app_env(Keys) of 81 | undefined -> 82 | Default; 83 | Value -> 84 | Value 85 | end. 86 | 87 | %% @doc Get a key out of the app.config file. You specify a nested 88 | %% key using a list of atoms, e.g. [riak_kv, storage_backend]. 89 | %% @equiv get_app_env(Keys, undefined) 90 | -spec get_app_env([atom()]) -> undefined | term(). 91 | get_app_env(Keys) -> 92 | {ok, Env} = application:get_env(riaknostic, app_config), 93 | find_nested_key(Keys, Env). 94 | 95 | %% @doc Get an -env flag out of the vm.args file. 96 | -spec get_vm_env(string()) -> string() | undefined. 97 | get_vm_env(Key) -> 98 | case application:get_env(riaknostic, vm_env) of 99 | undefined -> 100 | undefined; 101 | {ok, PList} -> 102 | proplists:get_value(Key, PList) 103 | end. 104 | 105 | %% @doc Determines the user/uid that the installed Riak runs as. 106 | -spec user() -> string(). 107 | user() -> 108 | case application:get_env(riaknostic, user) of 109 | {ok, Value} -> 110 | Value; 111 | _ -> undefined 112 | end. 113 | 114 | %% @doc The base directory from which the Riak script runs. 115 | -spec base_dir() -> file:filename(). 116 | base_dir() -> 117 | case application:get_env(riaknostic, base) of 118 | undefined -> 119 | filename:absname("."); 120 | {ok, Path} -> 121 | filename:absname(Path) 122 | end. 123 | 124 | %% @doc The Riak configuration directory. 125 | -spec etc_dir() -> file:filename(). 126 | etc_dir() -> 127 | case application:get_env(riaknostic, etc) of 128 | undefined -> 129 | filename:absname("./etc", base_dir()); 130 | {ok, Path} -> 131 | filename:absname(Path, base_dir()) 132 | end. 133 | 134 | %% @doc The local Riak node name. Includes whether the node uses short 135 | %% or long nodenames for distributed Erlang. 136 | -spec node_name() -> {shortnames | longnames, Name::string()}. 137 | node_name() -> 138 | case application:get_env(riaknostic, node_name) of 139 | undefined -> 140 | undefined; 141 | {ok, Node} -> 142 | Node 143 | end. 144 | 145 | %% @doc The Riak node's distributed Erlang cookie. 146 | -spec cookie() -> atom(). 147 | cookie() -> 148 | case application:get_env(riaknostic, cookie) of 149 | undefined -> 150 | undefined; 151 | {ok, Cookie} -> 152 | list_to_atom(Cookie) 153 | end. 154 | 155 | %% Private functions 156 | start_lager() -> 157 | application:load(lager), 158 | case application:get_env(riaknostic, log_level) of 159 | undefined -> 160 | {error, "Log level not set!"}; 161 | {ok, Level} -> 162 | application:set_env(lager, crash_log, undefined), 163 | application:set_env(lager, handlers, [{lager_console_backend, Level}]), 164 | lager:start() 165 | end. 166 | 167 | load_app_config() -> 168 | {ok, [[AppConfig]]} = init:get_argument(config), 169 | case file:consult(AppConfig) of 170 | {ok, [Config]} -> 171 | application:set_env(riaknostic, app_config, Config); 172 | _ -> 173 | {error, io_lib:format("Riak config file ~s is malformed!", [AppConfig])} 174 | end. 175 | 176 | load_vm_args() -> 177 | VmArgs = case init:get_argument(vm_args) of 178 | {ok, [[X]]} -> X; 179 | _ -> 180 | %% This is a backup. If for some reason -vm_args isn't specified 181 | %% then assume it lives in the same dir as app.config 182 | {ok, [[AppConfig]]} = init:get_argument(config), 183 | AppIndex = string:str(AppConfig, "app"), 184 | ConfigIndex = string:rstr(AppConfig, "config"), 185 | string:sub_string(AppConfig, 1, AppIndex - 1) ++ "vm" ++ 186 | string:sub_string(AppConfig, AppIndex + 3, ConfigIndex-1) ++ "args" 187 | end, 188 | 189 | case file:read_file(VmArgs) of 190 | {error, Reason} -> 191 | {error, io_lib:format("Could not read ~s, received error ~w!", [VmArgs, Reason])}; 192 | {ok, Binary} -> 193 | load_vm_args(Binary) 194 | end. 195 | 196 | load_vm_args(Bin) when is_binary(Bin) -> 197 | load_vm_args(re:split(Bin, "\s*\r?\n\s*", [{return, list}, trim])); 198 | load_vm_args([]) -> 199 | ok; 200 | load_vm_args([[$#|_]|T]) -> 201 | load_vm_args(T); 202 | load_vm_args([""|T]) -> 203 | load_vm_args(T); 204 | load_vm_args(["-sname " ++ NodeName|T]) -> 205 | application:set_env(riaknostic, node_name, {shortnames, string:strip(NodeName)}), 206 | load_vm_args(T); 207 | load_vm_args(["-name " ++ NodeName|T]) -> 208 | application:set_env(riaknostic, node_name, {longnames, string:strip(NodeName)}), 209 | load_vm_args(T); 210 | load_vm_args(["-setcookie " ++ Cookie|T]) -> 211 | application:set_env(riaknostic, cookie, string:strip(Cookie)), 212 | load_vm_args(T); 213 | load_vm_args(["-env " ++ Env|T]) -> 214 | [Key, Value] = re:split(Env, "\s+", [{return, list}, trim]), 215 | add_or_insert_env(vm_env, {Key, Value}), 216 | load_vm_args(T); 217 | load_vm_args([[$+|EmuFlags]|T]) -> 218 | [Flag|Rest] = re:split(EmuFlags, "\s+", [{return,list}, trim]), 219 | add_or_insert_env(emu_flags, {[$+|Flag], Rest}), 220 | load_vm_args(T); 221 | load_vm_args([[$-|InitFlags]|T]) -> 222 | [Flag|Rest] = re:split(InitFlags, "\s+", [{return,list}, trim]), 223 | add_or_insert_env(init_flags, {[$-|Flag], Rest}), 224 | load_vm_args(T); 225 | load_vm_args([Line|_]) -> 226 | {error, io_lib:format("Erroneous line in vm.args: ~s", [Line])}. 227 | 228 | add_or_insert_env(Key, Value) -> 229 | case application:get_env(riaknostic, Key) of 230 | undefined -> 231 | application:set_env(riaknostic, Key, [Value]); 232 | {ok, List} -> 233 | application:set_env(riaknostic, Key, [Value|List]) 234 | end. 235 | 236 | find_nested_key(_, undefined) -> 237 | undefined; 238 | find_nested_key([], Val) -> 239 | Val; 240 | find_nested_key([Key|T], PList) -> 241 | find_nested_key(T, proplists:get_value(Key, PList)). 242 | 243 | %% Determine the data directory(ies) for the configured storage backend 244 | -spec data_directory(atom()) -> [ file:filename() ]. 245 | data_directory(riak_kv_bitcask_backend) -> 246 | [ get_app_env([bitcask, data_root]) ]; 247 | data_directory(riak_kv_eleveldb_backend) -> 248 | [ get_app_env([eleveldb, data_root]) ]; 249 | data_directory(merge_index_backend) -> 250 | [ get_app_env([merge_index, data_root]) ]; 251 | data_directory(riak_kv_innostore_backend) -> 252 | [ get_app_env([innostore, data_home_dir]), 253 | get_app_env([innostore, log_group_home_dir]) ]; 254 | data_directory(riak_kv_multi_backend) -> 255 | [ multi_data_directory(Backend) || 256 | Backend <- get_app_env([riak_kv, multi_backend]), 257 | element(2, Backend) =/= riak_kv_memory_backend ]; 258 | data_directory(_) -> %% Memory or unknown backend 259 | []. 260 | 261 | %% Extracts data paths from multi_backend config 262 | multi_data_directory({_, riak_kv_bitcask_backend, Props}) -> 263 | case proplists:get_value(data_root, Props) of 264 | undefined -> 265 | get_app_env([bitcask, data_root]); 266 | Path when is_list(Path) -> 267 | Path 268 | end; 269 | multi_data_directory({_, riak_kv_eleveldb_backend, Props}) -> 270 | case proplists:get_value(data_root, Props) of 271 | undefined -> 272 | get_app_env([eleveldb, data_root]); 273 | Path when is_list(Path) -> 274 | Path 275 | end; 276 | multi_data_directory({_, riak_kv_innostore_backend, Props}) -> 277 | case proplists:get_value(data_home_dir, Props) of 278 | undefined -> 279 | get_app_env([innostore, data_home_dir]); 280 | Path when is_list(Path) -> 281 | Path 282 | end. 283 | -------------------------------------------------------------------------------- /src/riaknostic_node.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riaknostic - automated diagnostic tools for Riak 4 | %% 5 | %% Copyright (c) 2011 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | %% @doc Functions that help diagnostics interact with the local Riak 24 | %% node or other members of the cluster. 25 | -module(riaknostic_node). 26 | 27 | -export([can_connect/0, 28 | can_connect_all/0, 29 | stats/0, 30 | pid/0, 31 | local_command/2, 32 | local_command/3, 33 | local_command/4, 34 | cluster_command/2, 35 | cluster_command/3, 36 | cluster_command/4 37 | ]). 38 | 39 | %% @doc Calls the given 0-arity module and function on the local Riak 40 | %% node and returns the result of that call. 41 | %% @equiv local_command(Module, Function, []) 42 | %% @see can_connect/0. 43 | -spec local_command(Module::atom(), Function::atom()) -> term(). 44 | local_command(Module, Function) -> 45 | local_command(Module, Function, []). 46 | 47 | %% @doc Calls the given module and function with the given arguments 48 | %% on the local Riak node and returns the result of that call. 49 | %% @equiv local_command(Module, Function, Args, 5000) 50 | %% @see can_connect/0 51 | -spec local_command(Module::atom(), Function::atom(), Args::[term()]) -> term(). 52 | local_command(Module, Function, Args) -> 53 | local_command(Module, Function, Args, 5000). 54 | 55 | %% @doc Calls the given module and function with the given arguments 56 | %% on the local Riak node and returns the result of that call, 57 | %% returning an error if the call doesn't complete within the given 58 | %% timeout. 59 | %% @equiv rpc:call(RiakNodeName, Module, Function, Args, Timeout) 60 | %% @see can_connect/0 61 | -spec local_command(Module::atom(), Function::atom(), Args::[term()], Timeout::integer()) -> term(). 62 | local_command(Module, Function, Args, Timeout) -> 63 | riaknostic_util:log(debug, "Local RPC: ~p:~p(~p) [~p]", [Module, Function, Args, Timeout]), 64 | rpc:call(nodename(), Module, Function, Args, Timeout). 65 | 66 | %% @doc Calls the given 0-arity module and function on all members of 67 | %% the Riak cluster. 68 | %% @equiv cluster_command(Module, Function, []) 69 | %% @see can_connect/0 70 | -spec cluster_command(Module::atom(), Function::atom()) -> term(). 71 | cluster_command(Module, Function) -> 72 | cluster_command(Module, Function, []). 73 | 74 | %% @doc Calls the given module and function with the given arguments 75 | %% on all members of the Riak cluster. 76 | %% @equiv cluster_command(Module, Function, Args, 5000) 77 | %% @see can_connect/0 78 | -spec cluster_command(Module::atom(), Function::atom(), Args::[term()]) -> term(). 79 | cluster_command(Module, Function, Args) -> 80 | cluster_command(Module, Function, Args, 5000). 81 | 82 | %% @doc Calls the given module and function with the given arguments 83 | %% on all members for the Riak cluster, returning an error if the call 84 | %% doesn't complete within the given timeout. 85 | %% @equiv rpc:multicall(RiakClusterMembers, Module, Function, Args, Timeout) 86 | %% @see can_connect/0 87 | -spec cluster_command(Module::atom(), Function::atom(), Args::[term()], Timeout::integer()) -> term(). 88 | cluster_command(Module, Function, Args, Timeout) -> 89 | riaknostic_util:log(debug, "Cluster RPC: ~p:~p(~p) [~p]", [Module, Function, Args, Timeout]), 90 | Stats = stats(), 91 | {ring_members, RingMembers} = lists:keyfind(ring_members, 1, Stats), 92 | rpc:multicall(RingMembers, Module, Function, Args, Timeout). 93 | 94 | %% @doc Retrieves the operating system's process ID of the local Riak 95 | %% node. 96 | %% @equiv local_command(os, getpid) 97 | %% @see can_connect/0 98 | -spec pid() -> string(). 99 | pid() -> 100 | local_command(os, getpid). 101 | 102 | %% @doc Attempts to connect to the local Riak node if it is not 103 | %% already, and returns whether connection was successful. 104 | -spec can_connect() -> true | false. 105 | can_connect() -> 106 | case is_connected() of 107 | true -> true; 108 | false -> 109 | riaknostic_util:log(debug, "Not connected to the local Riak node, trying to connect. alive:~p connect_failed:~p", [is_alive(), connect_failed()]), 110 | maybe_connect() 111 | end. 112 | 113 | -spec can_connect_all() -> true | false. 114 | can_connect_all() -> 115 | case is_connected() of 116 | true -> 117 | case riaknostic_check_nodes_connected:check() of 118 | [] -> true; 119 | _ -> false 120 | end; 121 | false -> false 122 | end. 123 | 124 | %% @doc Fetches or returns previously fetched Riak statistics. 125 | %% @see can_connect/0 126 | -spec stats() -> [proplists:property()]. 127 | stats() -> 128 | case has_stats() of 129 | {ok, Stats} -> Stats; 130 | _ -> fetch_stats() 131 | end. 132 | 133 | %% Private functions 134 | is_connected() -> 135 | is_alive() andalso connect_failed() =/= true. 136 | 137 | maybe_connect() -> 138 | case connect_failed() of 139 | true -> false; 140 | _ -> try_connect() 141 | end. 142 | 143 | try_connect() -> 144 | TargetNode = nodename(), 145 | case is_alive() of 146 | true -> ok; 147 | _ -> start_net() 148 | end, 149 | case {net_kernel:hidden_connect_node(TargetNode), net_adm:ping(TargetNode)} of 150 | {true, pong} -> 151 | application:set_env(riaknostic, connect_failed, false), 152 | riaknostic_util:log(debug, "Connected to local Riak node ~p.", [TargetNode]), 153 | true; 154 | _ -> 155 | application:set_env(riaknostic, connect_failed, true), 156 | lager:warning("Could not connect to the local Riak node ~p, some checks will not run.", [TargetNode]), 157 | false 158 | end. 159 | 160 | connect_failed() -> 161 | case application:get_env(riaknostic, connect_failed) of 162 | {ok, true} -> true; 163 | undefined -> undefined; 164 | _ -> false 165 | end. 166 | 167 | start_net() -> 168 | riaknostic_util:log(debug, "Starting distributed Erlang."), 169 | {Type, RiakName} = riaknostic_config:node_name(), 170 | ThisNode = append_node_suffix(RiakName, "_diag"), 171 | {ok, _} = net_kernel:start([ThisNode, Type]), 172 | erlang:set_cookie(node(), riaknostic_config:cookie()). 173 | 174 | nodename() -> 175 | {_, Name} = riaknostic_config:node_name(), 176 | case string:tokens(Name, "@") of 177 | [_Node, _Host] -> 178 | list_to_atom(Name); 179 | [Node] -> 180 | [_, Host] = string:tokens(atom_to_list(node()), "@"), 181 | list_to_atom(lists:concat([Node, "@", Host])) 182 | end. 183 | 184 | append_node_suffix(Name, Suffix) -> 185 | case string:tokens(Name, "@") of 186 | [Node, Host] -> 187 | list_to_atom(lists:concat([Node, Suffix, os:getpid(), "@", Host])); 188 | [Node] -> 189 | list_to_atom(lists:concat([Node, Suffix, os:getpid()])) 190 | end. 191 | 192 | has_stats() -> 193 | case application:get_env(riaknostic, local_stats) of 194 | {ok, {badrpc, _Reason}} -> 195 | false; 196 | {ok, Stats} -> 197 | {ok, Stats}; 198 | undefined -> 199 | false 200 | end. 201 | 202 | fetch_stats() -> 203 | riaknostic_util:log(debug, "Fetching local riak_kv_status."), 204 | case local_command(riak_kv_status, statistics) of 205 | [] -> []; 206 | PList -> 207 | application:set_env(riaknostic, local_stats, PList), 208 | PList 209 | end. 210 | 211 | -------------------------------------------------------------------------------- /src/riaknostic_util.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riaknostic - automated diagnostic tools for Riak 4 | %% 5 | %% Copyright (c) 2011 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | %% @doc Utility functions for riaknostic. 24 | %% @end 25 | -module(riaknostic_util). 26 | -export([short_name/1, 27 | run_command/1, 28 | log/2,log/3, 29 | binary_to_float/1]). 30 | 31 | %% @doc Converts a check module name into a short name that can be 32 | %% used to refer to a check on the command line. For example, 33 | %% riaknostic_check_disk becomes "disk". 34 | -spec short_name(module()) -> iodata() | unicode:charlist(). 35 | short_name(Mod) when is_atom(Mod) -> 36 | re:replace(atom_to_list(Mod), "riaknostic_check_", "", [{return, list}]). 37 | 38 | %% @doc Runs a shell command and returns the output. stderr is 39 | %% redirected to stdout so its output will be included. 40 | -spec run_command(Command::iodata()) -> StdOut::iodata(). 41 | run_command(Command) -> 42 | riaknostic_util:log(debug, "Running shell command: ~s", [Command]), 43 | Port = erlang:open_port({spawn,Command},[exit_status, stderr_to_stdout]), 44 | do_read(Port, []). 45 | 46 | do_read(Port, Acc) -> 47 | receive 48 | {Port, {data, StdOut}} -> 49 | riaknostic_util:log(debug, "Shell command output: ~n~s~n",[StdOut]), 50 | do_read(Port, Acc ++ StdOut); 51 | {Port, {exit_status, _}} -> 52 | %%port_close(Port), 53 | Acc; 54 | Other -> 55 | io:format("~w", [Other]), 56 | do_read(Port, Acc) 57 | end. 58 | 59 | %% @doc Converts a binary containing a text representation of a float 60 | %% into a float type. 61 | -spec binary_to_float(binary()) -> float(). 62 | binary_to_float(Bin) -> 63 | list_to_float(binary_to_list(Bin)). 64 | 65 | log(Level, Format, Terms) -> 66 | case should_log(Level) of 67 | true -> 68 | io:format(lists:concat(["[", Level, "] ", Format, "~n"]), Terms); 69 | false -> 70 | ok 71 | end, 72 | lager:log(Level, self(), Format, Terms). 73 | 74 | log(Level, String) -> 75 | case should_log(Level) of 76 | true -> 77 | io:format(lists:concat(["[", Level, "] ", String, "~n"])); 78 | false -> 79 | ok 80 | end, 81 | lager:log(Level, self(), String). 82 | 83 | should_log(Level) -> 84 | AppLevel = case application:get_env(riaknostic, log_level) of 85 | undefined -> info; 86 | {ok, L0} -> L0 87 | end, 88 | lager_util:level_to_num(AppLevel) >= lager_util:level_to_num(Level). 89 | 90 | --------------------------------------------------------------------------------