├── .gitignore ├── LICENSE ├── README.md └── key_list_util.erl /.gitignore: -------------------------------------------------------------------------------- 1 | .eunit 2 | deps 3 | *.o 4 | *.beam 5 | *.plt 6 | erl_crash.dump 7 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | riak-key-list-util 2 | ================== 3 | 4 | A Riak console utility script for per-vnode key counting, siblings logging and more. (For Riak 1.1.4 and above) 5 | 6 | ## Usage 7 | This script needs to be run from a Riak console, on one of the Riak nodes. All nodes must be up and ready to accept requests or the script will fail. Every run of the script will append its output to the output of any previous run, so it’s a good idea to clear out the previous results. 8 | 9 | 10 | 1. Clone this repository 11 | 12 | ``` 13 | git clone https://github.com/basho-labs/riak-key-list-util.git 14 | ``` 15 | 16 | 2. Attach to a Riak console, of a running node. (Don't forget to detach via `CTRL+D`, when you're done). 17 | (You can hit Enter so that a `>` prompt appears). 18 | 19 | ``` 20 | riak attach 21 | ``` 22 | 23 | 3. From the console, load and compile the `key_list_util.erl` script. 24 | Be sure to use the full absolute path to the file. Avoid `~/` shortcuts for user home directories. 25 | For example, assuming that the repository above was cloned into `/root/`, do the following at the Riak console: 26 | 27 | ```erlang 28 | compile:file("/root/riak-key-list-util/key_list_util.erl"). 29 | ``` 30 | 31 | 4. You can now run the script functions. (See sections below) 32 | For example, `key_list_util:count_all_keys()` logs per-bucket key counts for each partition for a given node. 33 | 34 | **Important:** Once launched from a Riak console, this script will be loaded on each node in the cluster, 35 | *and run locally*. That is, the output files will be produced on *each node in the cluster*. They can then be 36 | gathered together in one location via `scp` or similar mechanisms. 37 | 38 | ## Counting keys (per bucket) and Logging Siblings 39 | 40 | This script works by iterating over every object in every partition, on a single partition, or on a single bucket. In order to avoid doing this multiple times, the functionality for counting keys and logging objects that contain siblings is unified into the same function. The three variations of the function are: 41 | 42 | ```erlang 43 | key_list_util:count_all_keys(OutputDirectory). 44 | key_list_util:count_all_keys_for_bucket(OutputDirectory, Bucket). 45 | key_list_util:count_all_keys_for_vnode(OutputDirectory, Vnode). 46 | ``` 47 | 48 | Example usage: 49 | 50 | ```erlang 51 | key_list_util:count_all_keys("/tmp/"). 52 | key_list_util:count_all_keys_for_bucket("/tmp/", test). 53 | key_list_util:count_all_keys_for_vnode("/tmp/", 1438665674247607560106752257205091097473808596992). 54 | ``` 55 | 56 | On the Riak console, once the script has been compiled (for this example, the log files will be created in `/tmp/`), run: 57 | 58 | ```erlang 59 | key_list_util:count_all_keys("/tmp/"). 60 | ``` 61 | 62 | The output should be: 63 | 64 | ``` 65 | Scanning cluster... 66 | Done. 67 | ok 68 | ``` 69 | 70 | Outside of the Riak console, you should now be able to see the resulting log files, for each partition on a given node. 71 | 72 | ``` 73 | ls /tmp/ 74 | ... 75 | dev1@192.168.10.1-548063113999088594326381812268606132370974703616-counts.log 76 | dev1@192.168.10.1-548063113999088594326381812268606132370974703616-siblings.log 77 | ... 78 | ``` 79 | 80 | #### Using Key Counts for Verification 81 | The per-bucket key count logs discussed above can be used for data verification, for example to compare key counts before and after an upgrade. 82 | 83 | The following commands have to be run on *each* Riak node. (The example assumes that the keys were logged to `/tmp/v1/counts/` directory before upgrade, and to `/tmp/v2/counts/` after upgrade): 84 | 85 | 1. Sort the log files in the output directories: 86 | 87 | ```bash 88 | for file in /tmp/v1/counts/*-counts.log; do sort -o $file $file; done 89 | for file in /tmp/v2/counts/*-counts.log; do sort -o $file $file; done 90 | ``` 91 | 92 | 2. Use `diff` to compare the "before" and "after" directories. `diff` will return no results if the directories are the same. 93 | 94 | ```bash 95 | diff -r /tmp/v1/counts /tmp/v2/counts 96 | ``` 97 | 98 | The sorted files should be identical before and after an upgrade. If not, the `diff` command will list any differences in any of the log files. 99 | 100 | ### Bucket key counts files 101 | These output files will be named in the form of `[node name]-[partition number]-counts.log`. Their contents will be in CSV format 102 | `"bucket name",#keys`, one line per bucket. 103 | 104 | For example: 105 | 106 | ```erlang 107 | "test-bucket-one",100 108 | "test-bucket-two",150 109 | ``` 110 | 111 | ## Logging All Keys 112 | While per-bucket key counts will suffice for data verification before and after Riak upgrades, some operations require more stringent 113 | (and more time-intensive) verification methods. For data verification before and after *ring resizing*, a full log of buckets and keys 114 | is required. 115 | 116 | To log all bucket/key combinations (this will generate a log of keys for each partition on each node): 117 | 118 | ```erlang 119 | key_list_util:log_all_keys("/tmp/"). 120 | ``` 121 | 122 | The output should be: 123 | 124 | ``` 125 | Scanning cluster... 126 | Done. 127 | ok 128 | ``` 129 | 130 | Outside of the Riak console, you should now be able to see the resulting log files, for each partition on a given node. 131 | 132 | ``` 133 | ls /tmp/ 134 | ... 135 | dev1@192.168.10.1-548063113999088594326381812268606132370974703616-keys.log 136 | ... 137 | ``` 138 | 139 | Their contents will be in the CSV format, `"bucket name","key"`, one line per key. 140 | 141 | For example: 142 | 143 | ```erlang 144 | "test-bucket-one","key2" 145 | "test-bucket-one","key1" 146 | "test-bucket-two","key100" 147 | ``` 148 | 149 | As with key counting, this function also has two alternatives which take two arguments. One which executes on a single bucket: `key_list_util:log_all_keys_for_bucket(OutputDirectory, Bucket).` Bucket is either a bucket name binary, or a {BucketType, Bucket} pair of binaries. The second option is to run per vnode (partition): `key_list_util:log_all_keys_for_vnode(OutputDirectory, Vnode)`. 150 | 151 | 152 | #### Using Key Logs for Verification 153 | The keylogs generated by `log_all_keys()` are most useful when sorted. The standard Unix `sort` command can help with sorting the key logs, 154 | merging them all into one file, removing duplicates (so that only one entry will exist for each key/bucket combination, rather than `N` entries), 155 | and also splitting the merged file into smaller files for easier processing. 156 | 157 | The following commands have to be run on *each* Riak node. (The example assumes that the keys were logged to `/tmp/v1/keys/` directory before upgrade): 158 | 159 | 1. Sort the key log files in the output directories: 160 | 161 | ```bash 162 | for file in /tmp/v1/keys/*-keys.log; do sort -u -o $file $file; done 163 | ``` 164 | 165 | 2. You can then merge the files (`sort -m`), remove duplicates (`-u`), and optionally `split` the resulting huge file into more manageable 166 | chunks (of 500,000 lines each). 167 | 168 | ```bash 169 | sort -m -u /tmp/v1/keys/*-keys.log | split -l 500000 /tmp/v1/keys/node1-sorted-keys- 170 | ``` 171 | 172 | 3. The split chunked files will be named `..-aa`, `..-ab` and so on: 173 | 174 | ```bash 175 | ls /tmp/v1/keys/ 176 | ... 177 | node1-sorted-keys-aa 178 | node1-sorted-keys-ab 179 | node1-sorted-keys-ac 180 | ... 181 | ``` 182 | 183 | Note the use of `node1` in the split filenames template, this will help disambiguate the key files when they're copied from 184 | all nodes into a central location. 185 | 186 | 4. The sorted, de-duped and split files can now be copied from the various nodes to a central location, for processing, using `scp`: 187 | 188 | ```bash 189 | scp root@192.168.10.1:/tmp/v1/keys/*-sorted-keys* /tmp/whole-cluster-v1/ 190 | # .. repeat for each riak node 191 | ``` 192 | 193 | #### Processing and Verification of Sorted Key Logs 194 | Once the sorted key logs are copied to a central location via `scp` (see section above): 195 | 196 | 1. The files, copied from all of the nodes in the cluster, need to be merged and de-duplicated again, before they are useful for verification: 197 | 198 | ```bash 199 | sort -m -u /tmp/all-nodes-v1/*-sorted-keys* | split -l 500000 /tmp/cluster-v1/all-sorted-keys- 200 | ``` 201 | 202 | 2. The resulting `cluster-v1` directory can then be used for data verification with the `diff` command: 203 | 204 | ```bash 205 | diff /tmp/cluster-v1/ /tmp/cluster-v2/ 206 | ``` 207 | 208 | ### Finding size of all keys 209 | Finding the size of all keys works similarly to listing all keys. You can iterate over all partitions in the cluster, a single partition, or a single bucket. The six variations of the function are: 210 | 211 | ``` 212 | size_all_keys(OutputDir). 213 | size_all_keys(OutputDir, Options). 214 | size_all_keys_for_bucket(OutputDir, Bucket). 215 | size_all_keys_for_bucket(OutputDir, Bucket, Options). 216 | size_all_keys_for_vnode(OutputDir, Vnode). 217 | size_all_keys_for_vnode(OutputDir, Vnode, Options). 218 | ``` 219 | 220 | The output will appear in the form of `[node name]-[partition number]-sizes.log`. These functions can accept options as a list. If no options are required, you may leave off the argument. The `raw_size` option controls whether to return the whole object with metadata, or just the size of values. The `ignore_siblings` option controls whether to consider siblings in the calculations, or return the size of the first value (or an estimate of all the objects divided by the number of siblings). These two options are mutually exclusive. If both are provided, `raw_size` will take precedence, and the script will silently drop `ignore_siblings`. 221 | 222 | For example: 223 | 224 | ```erlang 225 | key_list_util:size_all_keys("/tmp/"). 226 | ``` 227 | 228 | Print out full object size, including any siblings: 229 | 230 | ```erlang 231 | key_list_util:size_all_keys("/tmp/", [raw_size]). 232 | ``` 233 | 234 | The `size_all_keys()` function requires key duplication similar to `log_all_keys()` above. Its output adds a size of objects in bytes to the end of the `log_all_keys()` output. 235 | 236 | ### Logs of Objects with Siblings 237 | These will be named in the form of `[node name]-[partition number]-siblings.log`. (If no objects with siblings are found for a particular partition, 238 | no siblings log file will be created for that partition). The files consist of the following entries for each object, separated by a single blank line: 239 | 240 | ```erlang 241 | {<<"bucket name">>,<<"object key">>,2} 242 | {{vtag,"6oEMJzrPKX6gPgPYm9s2pt"}, 243 | {date_modified,{{2014,5,14},{11,6,30}}}, 244 | {is_deleted,false}} 245 | <<"sibling value 2">> 246 | {{vtag,"6ZPSo6ATuiQohGuaBYsdWq"}, 247 | {date_modified,{{2014,5,14},{11,6,25}}}, 248 | {is_deleted,false}} 249 | <<"sibling value 1">> 250 | ``` 251 | 252 | The siblings (each with their own `vtag`), will be sorted by timestamp, most recent to least. (The `2` following the bucket name and key in the example above denotes the number of siblings for that object). 253 | 254 | #### Note on Tombstones 255 | In the sibling logs above, tombstones will be denoted by `{is_deleted,true}`. 256 | 257 | ## Sibling Resolution 258 | To force-reconcile all siblings on the cluster, use the `resolve_all_siblings()` function. As with the key counting function above, 259 | the script will run on every node in the cluster (sequentially, this time). 260 | 261 | On the Riak console, once the script has been compiled (for this example, the log files will be created in `/tmp/`), run: 262 | 263 | ```erlang 264 | key_list_util:resolve_all_siblings("/tmp/"). 265 | ``` 266 | 267 | The output files this function produces will be similar to the `*-siblings.log` files above, with the addition of a 268 | `Resolved to Vtag:` line at the end of each object listing. For example: 269 | 270 | ```erlang 271 | {<<"bucket name">>,<<"object key">>,2} 272 | {{vtag,"6oEMJzrPKX6gPgPYm9s2pt"}, 273 | {date_modified,{{2014,5,14},{11,6,30}}}, 274 | {is_deleted,false}} 275 | <<"sibling value 2">> 276 | {{vtag,"6ZPSo6ATuiQohGuaBYsdWq"}, 277 | {date_modified,{{2014,5,14},{11,6,25}}}, 278 | {is_deleted,false}} 279 | <<"sibling value 1">> 280 | Resolved to Vtag: "6oEMJzrPKX6gPgPYm9s2pt" 281 | ``` 282 | 283 | Note: There will be fewer `*-siblings.log` entries than during key counting and sibling logging. Because the sibling resolution 284 | runs sequentially, once a sibling is resolved for one node, this also resolves all 3 replicas, and so the resolved objects will not be 285 | encountered on other nodes. 286 | 287 | As with the other functions in this utility, this function also has two alternatives. The first two-argument version resolves siblings for only a single bucket: `key_list_util:resolve_all_siblings_for_bucket(OutputDirectory, Bucket).` Here, Bucket is either a bucket name binary, or a {BucketType, Bucket} pair of binaries. The second option is to run per vnode (partition): `key_list_util:resolve_all_keys_for_vnode(OutputDirectory, Vnode).` 288 | 289 |
290 | 291 | ## Direct Deletes 292 | 293 | Rarely, you will come upon an object that has been turned into a tombstone, but has not been reaped. This can happen due to the distributed nature of the database coupled with it being an AP, rather than CP store. Also, the erlang timers that schedule the reaps can occasionally fail for other reasons. 294 | 295 | Whatever the case, the following function is there for use in such a case: 296 | 297 | ``` 298 | local_direct_delete(Index, Bucket, Key) 299 | ``` 300 | 301 | Here, **Index** is an integer representing the vnode or partition that the tombstone is on. **Bucket** and **Key** are binaries, as described above, and look like *\<\<"bucketOrKeyName"\>\>*. 302 | 303 |
304 | 305 | ## Preflists 306 | 307 | It can be useful to get the preflist, up to a specified n-val, for a given object identifier. One example of use is to determine whether an object has been orphaned at an n-val higher than that in the bucket config. This might occur due to a previous lowering of n-val for that bucket. 308 | 309 | ``` 310 | get_preflist_for_key(Bucket, Key, NValue) 311 | ``` 312 | 313 | Here, **Bucket** and **Key** are binaries as above. **NValue** is an integer representing the length of the preflist you want returned. 314 | 315 | ***Please note*** that results will differ for different ring sizes. So make sure you test in the same cluster you are working with, or one with the same ring size. Also, NValue cannot be higher than the ringsize of the cluster you are attached to; if it is, the function will fail with an error. 316 | 317 |
318 | 319 | \~ 320 | -------------------------------------------------------------------------------- /key_list_util.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% key_list_util: utility console script for per-vnode key counting, siblings logging and more 4 | %% 5 | %% Copyright (c) 2014 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | %% TODOs 24 | %% * replace foreach loops with foldl loops, taking errors into account and returning them 25 | 26 | -module(key_list_util). 27 | -compile(export_all). 28 | 29 | % Describes the Contents of a Riak object. A "sibling" is an instance of this record. 30 | % Duplicated from riak_kv/riak_object, since it's needed by compare_content_dates() 31 | -record(r_content, { 32 | metadata :: dict(), 33 | value :: term() 34 | }). 35 | 36 | % Describes a Riak Object 37 | % Duplicated from riak_kv/riak_object, since it's needed by compare_content_dates() 38 | -record(r_object, { 39 | bucket :: riak_object:bucket(), 40 | key :: riak_object:key(), 41 | contents :: [#r_content{}], 42 | vclock = vclock:fresh() :: vclock:vclock(), 43 | updatemetadata=dict:store(clean, true, dict:new()) :: dict(), 44 | updatevalue :: term() 45 | }). 46 | 47 | %% ================================================================================================= 48 | %% In the following functions, Bucket can be a Bucket or a {BucketType, Bucket} pair. 49 | %% Including this parameter implies the operation will be run only upon the specified Bucket or pair. 50 | 51 | count_all_keys(OutputDir) -> 52 | process_cluster_parallel(OutputDir, [count_keys, log_siblings]). 53 | 54 | count_all_keys_for_bucket(OutputDir, Bucket) -> 55 | process_cluster_parallel(OutputDir, [count_keys, log_siblings, {bucket, Bucket}]). 56 | 57 | count_all_keys_for_vnode(OutputDir, Vnode) -> 58 | process_cluster_parallel_for_vnode(OutputDir, [count_keys, log_siblings], Vnode). 59 | 60 | log_all_keys(OutputDir) -> 61 | process_cluster_parallel(OutputDir, [log_keys]). 62 | 63 | log_all_keys_for_bucket(OutputDir, Bucket) -> 64 | process_cluster_parallel(OutputDir, [log_keys, {bucket, Bucket}]). 65 | 66 | log_all_keys_for_vnode(OutputDir, Vnode) -> 67 | process_cluster_parallel_for_vnode(OutputDir, [log_keys], Vnode). 68 | 69 | % Find the size of data stored. Takes options to add object metadata and 70 | % count siblings. Default is to only count value sizes and only the first 71 | % returned sibling. Send Options as a list of atoms 72 | % e.g. [raw_size, ignore_siblings] to add metadata to results and count 73 | % all siblings 74 | size_all_keys(OutputDir) -> 75 | process_cluster_parallel(OutputDir, [size_keys]). 76 | 77 | size_all_keys(OutputDir, Options) -> 78 | process_cluster_parallel(OutputDir, [size_keys|Options]). 79 | 80 | size_all_keys_for_bucket(OutputDir, Bucket) -> 81 | process_cluster_parallel(OutputDir, [size_keys, {bucket, Bucket}]). 82 | 83 | size_all_keys_for_bucket(OutputDir, Bucket, Options) -> 84 | process_cluster_parallel(OutputDir, [size_keys, {bucket, Bucket}] ++ Options). 85 | 86 | size_all_keys_for_vnode(OutputDir, Vnode) -> 87 | process_cluster_parallel_for_vnode(OutputDir, [size_keys], Vnode). 88 | 89 | size_all_keys_for_vnode(OutputDir, Vnode, Options) -> 90 | process_cluster_parallel_for_vnode(OutputDir, [size_keys|Options], Vnode). 91 | 92 | % SleepPeriod - optional amount of time to sleep between each key operation, 93 | % in milliseconds 94 | log_all_keys(OutputDir, SleepPeriod) -> 95 | process_cluster_parallel(OutputDir, [log_keys, {sleep_for, SleepPeriod}]). 96 | 97 | log_all_keys_for_bucket(OutputDir, SleepPeriod, Bucket) -> 98 | process_cluster_parallel(OutputDir, [log_keys, {sleep_for, SleepPeriod}, {bucket, Bucket}]). 99 | 100 | log_all_keys_for_vnode(OutputDir, SleepPeriod, Vnode) -> 101 | process_cluster_parallel_for_vnode(OutputDir, [log_keys, {sleep_for, SleepPeriod}], Vnode). 102 | 103 | %% Call size_all_keys* functions like: 104 | %% size_all_keys_(OutputDir, [{sleep_for, SleepPeriod}]) 105 | %% to sleep between key operations. Setting Options already creates a /2 function 106 | %% Other options can also be added to the call list 107 | 108 | resolve_all_siblings(OutputDir) -> 109 | process_cluster_serial(OutputDir, [log_siblings, resolve_siblings]). 110 | 111 | resolve_all_siblings_for_bucket(OutputDir, Bucket) -> 112 | process_cluster_serial(OutputDir, [log_siblings, resolve_siblings, {bucket, Bucket}]). 113 | 114 | resolve_all_siblings_for_vnode(OutputDir, Vnode) -> 115 | process_cluster_serial_for_vnode(OutputDir, [log_siblings, resolve_siblings], Vnode). 116 | 117 | local_direct_delete(Index, Bucket, Key) when 118 | is_integer(Index), 119 | is_binary(Bucket), 120 | is_binary(Key) -> 121 | DeleteReq = {riak_kv_delete_req_v1, {Bucket, Key}, make_ref()}, 122 | riak_core_vnode_master:sync_command({Index, node()}, DeleteReq, riak_kv_vnode_master). 123 | 124 | get_preflist_for_key(Bucket, Key, NValue) when 125 | is_binary(Bucket), 126 | is_binary(Key), 127 | is_integer(NValue) -> 128 | BKey = {Bucket,Key}, 129 | {ok, Ring} = riak_core_ring_manager:get_my_ring(), 130 | DocIdx = riak_core_util:chash_key(BKey), 131 | % BucketProps = riak_core_bucket:get_bucket(Bucket, Ring), 132 | % [NValue] = [Y || {X1, Y} <- BucketProps, n_val == X1], 133 | UpNodes = riak_core_node_watcher:nodes(riak_kv), 134 | Preflist = riak_core_apl:get_apl_ann(DocIdx, NValue, Ring, UpNodes), 135 | [IndexNode || {IndexNode, _Type} <- Preflist]. 136 | 137 | 138 | %% ================================================================================================= 139 | 140 | 141 | % Used for sorting an object's siblings in modified timestamp order (most recently modified to least) 142 | % Duplicated from riak_kv/riak_object (since it's not exported from that module) 143 | compare_content_dates(C1, C2) -> 144 | D1 = dict:fetch(<<"X-Riak-Last-Modified">>, C1#r_content.metadata), 145 | D2 = dict:fetch(<<"X-Riak-Last-Modified">>, C2#r_content.metadata), 146 | %% true if C1 was modifed later than C2 147 | Cmp1 = riak_core_util:compare_dates(D1, D2), 148 | %% true if C2 was modifed later than C1 149 | Cmp2 = riak_core_util:compare_dates(D2, D1), 150 | %% check for deleted objects 151 | Del1 = dict:is_key(<<"X-Riak-Deleted">>, C1#r_content.metadata), 152 | Del2 = dict:is_key(<<"X-Riak-Deleted">>, C2#r_content.metadata), 153 | 154 | SameDate = (Cmp1 =:= Cmp2), 155 | case {SameDate, Del1, Del2} of 156 | {false, _, _} -> 157 | Cmp1; 158 | {true, true, false} -> 159 | false; 160 | {true, false, true} -> 161 | true; 162 | _ -> 163 | %% Dates equal and either both present or both deleted, compare 164 | %% by opaque contents. 165 | C1 < C2 166 | end. 167 | 168 | get_vtag(Obj) -> 169 | dict:fetch(<<"X-Riak-VTag">>, Obj#r_content.metadata). 170 | 171 | is_deleted(Obj) -> 172 | dict:is_key(<<"X-Riak-Deleted">>, Obj#r_content.metadata). 173 | 174 | % Loads the contents of a module (this module, usually) on every node in the cluster, 175 | % to parallelize and cut down on inter-node disterl chatter. 176 | load_module_on_nodes(Module, Nodes) -> 177 | rpc:multicall(Nodes, code, purge, [Module]), 178 | case code:get_object_code(Module) of 179 | {Module, Bin, File} -> 180 | {_, []} = rpc:multicall(Nodes, code, load_binary, [Module, File, Bin]); 181 | error -> 182 | error(lists:flatten(io_lib:format("unable to get_object_code(~s)", [Module]))) 183 | end. 184 | 185 | % Log the vtag, value and deleted status of a given sibling object 186 | log_sibling_contents(Obj, OutputFilename) -> 187 | DateModified = calendar:now_to_local_time(dict:fetch(<<"X-Riak-Last-Modified">>, Obj#r_content.metadata)), 188 | Deleted = is_deleted(Obj), 189 | Vtag = get_vtag(Obj), 190 | Msg = io_lib:format("~p~n", [{{vtag, Vtag}, {date_modified, DateModified}, {is_deleted, Deleted}}]), 191 | file:write_file(OutputFilename, Msg, [append]), 192 | Value = Obj#r_content.value, 193 | file:write_file(OutputFilename, io_lib:format("~p~n", [Value]), [append]). 194 | 195 | % Returns the last (most recent) version of the object (by timestamp) 196 | % If the last version is a tombstone, return the 197 | % (The list of siblings is pre-sorted by timestamp, most recent to least) 198 | last_valid_vtag([]) -> 199 | {error, "No valid (non-deleted) sibling found."}; 200 | last_valid_vtag([Sibling|Rest]) -> 201 | case is_deleted(Sibling) of 202 | false -> 203 | {ok, {get_vtag(Sibling), Sibling}}; 204 | % {ok, {test, test}}; 205 | true -> 206 | last_valid_vtag(Rest) 207 | end. 208 | 209 | resolve_object_siblings(OutputFilename, Bucket, Key, SiblingsByDate) -> 210 | case last_valid_vtag(SiblingsByDate) of 211 | {ok, {CorrectVtag, CorrectSibling}} -> 212 | case force_reconcile(Bucket, Key, CorrectSibling) of 213 | ok -> 214 | Msg = io_lib:format("Resolved to Vtag: ~p~n", [CorrectVtag]); 215 | {error, Error} -> 216 | Msg = io_lib:format("Error resolving to Vtag ~p :: ~p~n", [CorrectVtag, Error]) 217 | end; 218 | {error, Error} -> 219 | Msg = io_lib:format("Error resolving siblings: ~p~n", [{Error}]) 220 | end, 221 | file:write_file(OutputFilename, Msg, [append]). 222 | 223 | force_reconcile(Bucket, Key, CorrectSibling) -> 224 | {ok, C} = riak:local_client(), 225 | {ok, OldObj} = C:get(Bucket, Key), 226 | NewObj = riak_object:update_metadata(riak_object:update_value(OldObj, CorrectSibling#r_content.value), CorrectSibling#r_content.metadata), 227 | UpdatedObj = riak_object:apply_updates(NewObj), 228 | C:put(UpdatedObj, all, all). % W=all, DW=all 229 | 230 | % Convert a serialized binary into a riak_object() record 231 | % The function riak_object:from_binary() was introduced in 1.4, so 232 | % we need to check for its existence and use it if possible 233 | unserialize(_B, _K, Val = #r_object{}) -> 234 | Val; 235 | unserialize(_B, _K, <<131, _Rest/binary>>=Val) -> 236 | binary_to_term(Val); 237 | unserialize(Bucket, Key, Val) -> 238 | try 239 | riak_object:from_binary(Bucket, Key, Val) 240 | catch _:_ -> 241 | {error, bad_object_format} 242 | end. 243 | 244 | % Log a key/bucket pair to a file 245 | log_key(OutputFilename, Bucket, Key, Options) -> 246 | Msg = io_lib:format("~p,~s~n", [Bucket, binary_to_list(Key)]), 247 | file:write_file(OutputFilename, Msg, [append]), 248 | case lists:keyfind(sleep_for, 1, Options) of 249 | {sleep_for, SleepPeriod} -> 250 | timer:sleep(SleepPeriod); 251 | _ -> ok 252 | end. 253 | 254 | % Log a key/bucket pair to a file with object sizes 255 | size_key(OutputFilename, Bucket, Key, ObjBinary, Options) -> 256 | case {lists:member(raw_size, Options), lists:member(ignore_siblings, Options)} of 257 | {true, _} -> % raw_size, size of the object, including siblings 258 | ByteSize = byte_size(ObjBinary); 259 | {false, true} -> % ignore_siblings, size of first value returned 260 | [Value|_] = riak_object:get_values(unserialize(Bucket, Key, ObjBinary)), 261 | ByteSize = byte_size(Value); 262 | {false, false} -> % neither, aggregates values for all siblings 263 | % Get siblings 264 | ByteSize = lists:foldl(fun(X,Sum) -> byte_size(X) + Sum end, 0, riak_object:get_values(unserialize(Bucket, Key, ObjBinary))) 265 | end, 266 | Msg = io_lib:format("~p,~s,~p~n", [Bucket, binary_to_list(Key), ByteSize]), 267 | file:write_file(OutputFilename, Msg, [append]), 268 | case lists:keyfind(sleep_for, 1, Options) of 269 | {sleep_for, SleepPeriod} -> 270 | timer:sleep(SleepPeriod); 271 | _ -> ok 272 | end. 273 | 274 | % Log all siblings for a riak object (if any exist). 275 | log_or_resolve_siblings(OutputFilename, Bucket, Key, ObjBinary, Options) -> 276 | Obj = unserialize(Bucket, Key, ObjBinary), 277 | SiblingCount = case Obj of 278 | {error, _Error} -> 279 | 1; % Error unserializing, skip the logging of the sibling count, below 280 | _ -> 281 | riak_object:value_count(Obj) 282 | end, 283 | 284 | if SiblingCount > 1 -> 285 | Contents = Obj#r_object.contents, 286 | SiblingsByDate = lists:sort(fun compare_content_dates/2, Contents), 287 | Msg = io_lib:format("~n~p~n", [{Bucket, Key, SiblingCount}]), 288 | file:write_file(OutputFilename, Msg, [append]), 289 | 290 | lists:foreach(fun(Sibling) -> log_sibling_contents(Sibling, OutputFilename) end, SiblingsByDate), 291 | 292 | case lists:member(resolve_siblings, Options) of 293 | true -> 294 | resolve_object_siblings(OutputFilename, Bucket, Key, SiblingsByDate); 295 | _ -> ok 296 | end; 297 | true -> ok 298 | end. 299 | 300 | member_nodes() -> 301 | {ok, Ring} = riak_core_ring_manager:get_raw_ring(), 302 | riak_core_ring:all_members(Ring). 303 | 304 | % For each node in the cluster, in parallel, load this module, 305 | % and invoke the process_node() function on its vnodes. 306 | process_cluster_parallel(OutputDir, Options) -> 307 | io:format("Scanning all nodes in parallel...~n"), 308 | Members = member_nodes(), 309 | load_module_on_nodes(?MODULE, Members), 310 | rpc:multicall(Members, ?MODULE, process_node, [OutputDir, Options]), 311 | io:format("Done.~n"). 312 | 313 | % For each node in the cluster, load this module, 314 | % and invoke the process_node() function on its vnodes. 315 | process_cluster_serial(OutputDir, Options) -> 316 | io:format("Scanning all nodes serially...~n"), 317 | Members = member_nodes(), 318 | load_module_on_nodes(?MODULE, Members), 319 | NodeFun = fun(Node) -> 320 | io:format("Processing node ~p~n", [Node]), 321 | rpc:call(Node, ?MODULE, process_node, [OutputDir, Options]) 322 | end, 323 | lists:foreach(NodeFun, Members), 324 | io:format("Done.~n"). 325 | 326 | %% For per vnode operations 327 | process_cluster_parallel_for_vnode(OutputDir, Options, Vnode) -> 328 | io:format("Scanning all nodes in parallel...~n"), 329 | Members = member_nodes(), 330 | load_module_on_nodes(?MODULE, Members), 331 | rpc:multicall(Members, ?MODULE, process_vnode_on_node, [OutputDir, Options, Vnode]), 332 | io:format("Done.~n"). 333 | 334 | % For each node in the cluster, load this module, 335 | % and invoke the process_node() function on its vnodes. 336 | process_cluster_serial_for_vnode(OutputDir, Options, Vnode) -> 337 | io:format("Scanning all nodes serially...~n"), 338 | Members = member_nodes(), 339 | load_module_on_nodes(?MODULE, Members), 340 | NodeFun = fun(Node) -> 341 | io:format("Processing node ~p~n", [Node]), 342 | rpc:call(Node, ?MODULE, process_vnode_on_node, [OutputDir, Options, Vnode]) 343 | end, 344 | lists:foreach(NodeFun, Members), 345 | io:format("Done.~n"). 346 | 347 | % Invoked on each member node in the ring 348 | % Calls process_vnode() on each vnode local to this node. 349 | process_node(OutputDir, Options) -> 350 | {ok, Ring} = riak_core_ring_manager:get_raw_ring(), 351 | Owners = riak_core_ring:all_owners(Ring), 352 | LocalVnodes = [IdxOwner || IdxOwner={_, Owner} <- Owners, 353 | Owner =:= node()], 354 | lists:foreach(fun(Vnode) -> process_vnode(Vnode, OutputDir, Options) end, LocalVnodes). %% TODO foreach 355 | 356 | % Performs a riak_kv_vnode:fold(), and invokes logging functions for each key in this partition 357 | process_vnode(Vnode, OutputDir, Options) -> 358 | {Partition, Node} = Vnode, 359 | CountsFilename = filename:join(OutputDir, [io_lib:format("~s-~p-counts.log", [Node, Partition])]), 360 | SiblingsFilename = filename:join(OutputDir, [io_lib:format("~s-~p-siblings.log", [Node, Partition])]), 361 | KeysFilename = filename:join(OutputDir, [io_lib:format("~s-~p-keys.log", [Node, Partition])]), 362 | SizesFilename = filename:join(OutputDir, [io_lib:format("~s-~p-sizes.log", [Node, Partition])]), 363 | 364 | FoldOptions = case proplists:get_value(bucket, Options, undefined) of 365 | undefined -> 366 | []; 367 | SingleBucket -> 368 | [{bucket, SingleBucket}] 369 | end, 370 | 371 | InitialAccumulator = dict:store(<<"BucketKeyCounts">>, dict:new(), dict:new()), 372 | ProcessObj = fun(BKey, ObjBinary, AccDict) -> 373 | {Bucket, Key} = BKey, 374 | 375 | case lists:member(log_keys, Options) of 376 | true -> 377 | log_key(KeysFilename, Bucket, Key, Options); 378 | _ -> ok 379 | end, 380 | 381 | case lists:member(size_keys, Options) of 382 | true -> 383 | size_key(SizesFilename, Bucket, Key, ObjBinary, Options); 384 | _ -> ok 385 | end, 386 | 387 | case lists:member(log_siblings, Options) of 388 | true -> 389 | log_or_resolve_siblings(SiblingsFilename, Bucket, Key, ObjBinary, Options); 390 | _ -> ok 391 | end, 392 | 393 | case lists:member(count_keys, Options) of 394 | true -> 395 | % Update per-bucket key count 396 | CountDict = dict:update_counter(Bucket, 1, dict:fetch(<<"BucketKeyCounts">>, AccDict)), 397 | dict:store(<<"BucketKeyCounts">>, CountDict, AccDict); 398 | _ -> AccDict 399 | end 400 | end, 401 | Results = riak_kv_vnode:fold(Vnode, ProcessObj, InitialAccumulator, FoldOptions), 402 | write_vnode_totals(CountsFilename, Results). 403 | 404 | match_vnode(Vnode, VnodeArg, OutputDir, Options) -> 405 | % 406 | %io:format("Comparing ~p~n", [VnodeArg]), 407 | %io:format("With ~p~n", [Vnode]), 408 | case is_element_of_tuple(Vnode, VnodeArg) of 409 | true -> 410 | io:format("Processing vnode ~p~n", [VnodeArg]), 411 | process_vnode(Vnode, OutputDir, Options); 412 | _ -> ok 413 | end. 414 | 415 | 416 | % Invoked on each member node in the ring 417 | % Calls process_vnode() on each vnode local to this node. 418 | process_vnode_on_node(OutputDir, Options, VnodeArg) -> 419 | {ok, Ring} = riak_core_ring_manager:get_raw_ring(), 420 | Owners = riak_core_ring:all_owners(Ring), 421 | LocalVnodes = [IdxOwner || IdxOwner={_, Owner} <- Owners, Owner =:= node()], 422 | lists:foreach(fun(Vnode) -> match_vnode(Vnode, VnodeArg, OutputDir, Options) end, LocalVnodes). 423 | 424 | 425 | 426 | is_element_of_tuple(Tuple, Element) -> 427 | lists:member(Element, tuple_to_list(Tuple)). 428 | 429 | write_vnode_totals(OutputFilename, Results) -> 430 | case dict:is_key(<<"BucketKeyCounts">>, Results) of 431 | true -> 432 | Counts = dict:to_list(dict:fetch(<<"BucketKeyCounts">>, Results)), 433 | WriteBucketFun = fun(BucketCount) -> 434 | {Bucket, Count} = BucketCount, 435 | file:write_file(OutputFilename, io_lib:format("~p,~B~n", [Bucket, Count]), [append]) 436 | end, 437 | lists:foreach(WriteBucketFun, Counts); %% TODO foreach 438 | _ -> ok %% TODO return a result 439 | end. 440 | --------------------------------------------------------------------------------