├── .github └── workflows │ └── erlang.yml ├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── RELNOTES.md ├── build └── publish ├── eqc └── bucket_props_codec_eqc.erl ├── include ├── riak_pb_kv_codec.hrl └── riak_ts_ttb.hrl ├── rebar.config ├── rebar3 ├── src ├── riak.proto ├── riak_dt.proto ├── riak_kv.proto ├── riak_pb.app.src ├── riak_pb_codec.erl ├── riak_pb_dt_codec.erl ├── riak_pb_kv_codec.erl ├── riak_pb_messages.csv ├── riak_pb_search_codec.erl ├── riak_pb_ts_codec.erl ├── riak_search.proto ├── riak_ts.proto ├── riak_ttb_codec.erl └── riak_yokozuna.proto └── test ├── encoding_test.erl └── riak_pb_dt_codec_tests.erl /.github/workflows/erlang.yml: -------------------------------------------------------------------------------- 1 | name: Erlang CI 2 | 3 | on: 4 | push: 5 | branches: [ develop ] 6 | pull_request: 7 | branches: [ develop ] 8 | 9 | 10 | jobs: 11 | 12 | build: 13 | 14 | runs-on: ubuntu-latest 15 | 16 | strategy: 17 | fail-fast: false 18 | matrix: 19 | otp: 20 | - "25.1" 21 | - "24.3" 22 | - "22.3" 23 | 24 | container: 25 | image: erlang:${{ matrix.otp }} 26 | 27 | steps: 28 | - uses: lukka/get-cmake@latest 29 | - uses: actions/checkout@v2 30 | - name: Compile 31 | run: ./rebar3 compile 32 | - name: Run xref and dialyzer 33 | run: ./rebar3 do xref, dialyzer 34 | - name: Run eunit 35 | run: ./rebar3 as gha do eunit 36 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Erlang 2 | .eunit/* 3 | deps/* 4 | priv/* 5 | *.o 6 | ebin/*.beam 7 | ebin/riak_pb.app 8 | include/*_pb.hrl 9 | src/*_pb.erl 10 | doc/* 11 | .qc 12 | .eqc-info 13 | current_counterexample.eqc 14 | src/riak_pb_messages.erl 15 | .local_dialyzer_plt 16 | dialyzer_unhandled_warnings 17 | dialyzer_warnings 18 | .rebar/* 19 | .rebar3/ 20 | _build/ 21 | .DS_Store 22 | test/*.beam 23 | rebar.lock 24 | 25 | # Java 26 | target/* 27 | 28 | #IntelliJ 29 | .idea/ 30 | riak_pb.iml 31 | 32 | # C 33 | c/* 34 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: compile rel cover test dialyzer 2 | REBAR=./rebar3 3 | 4 | compile: 5 | $(REBAR) compile 6 | 7 | clean: 8 | rm -rf include/*_pb.hrl 9 | rm -rf src/*_pb.erl 10 | $(REBAR) clean 11 | 12 | cover: test 13 | $(REBAR) cover 14 | 15 | test: compile 16 | $(REBAR) eunit 17 | 18 | dialyzer: 19 | $(REBAR) dialyzer 20 | 21 | xref: 22 | $(REBAR) xref 23 | 24 | check: test dialyzer xref 25 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Riak Protocol Buffers Messages 2 | 3 | [![Erlang CI Actions Status](https://github.com/basho/riak_pb/workflows/Erlang%20CI/badge.svg)](https://github.com/basho/riak_pb/actions) 4 | 5 | This repository contains the message definitions for the Protocol 6 | Buffers-based interface to [Riak](https://github.com/basho/riak) and 7 | various Erlang-specific utility modules for the message types. 8 | 9 | This is distributed separately from the Riak server and clients, 10 | allowing it to serve as an independent representation of the supported 11 | messages. Additionally, the `.proto` descriptions are broken out by 12 | functional area: 13 | 14 | * `riak.proto` contains "global" messages like the error message and 15 | the "server info" calls. 16 | * `riak_kv.proto` contains messages related to Riak KV. 17 | 18 | Other specifications may arise as more features are exposed via the PB 19 | interface. 20 | 21 | ## Protocol 22 | 23 | The Riak PBC protocol encodes requests and responses as Protocol 24 | Buffers messages. Each request message results in one or more 25 | response messages. As message type and length are not encoded by PB, 26 | they are sent on the wire as: 27 | 28 | 29 | 30 | * `length` is the length of `msg_code` (1 byte) plus the message length 31 | in bytes encoded in network order (big endian). 32 | 33 | * `msg_code` indicates what is encoded as `pbmsg` 34 | 35 | * `pbmsg` is the encoded protocol buffer message 36 | 37 | On connect, the client can make requests and will receive responses. 38 | For each request message there is a corresponding response message, or 39 | the server will respond with an error message if something has gone 40 | wrong. 41 | 42 | The client should be prepared to handle messages without any `pbmsg` 43 | (i.e. `length == 1`) for requests where the response is simply an 44 | acknowledgment. 45 | 46 | In some cases, a client may receive multiple response messages for a 47 | single request. The response message will typically include a boolean 48 | `done` field that signifies the last message in a sequence. 49 | 50 | ### Registered Message Codes 51 | 52 | [Message codes](http://docs.basho.com/riak/latest/dev/references/protocol-buffers/#Message-Codes) and documentation can be found in the protocol-buffers 53 | [section](http://docs.basho.com/riak/latest/dev/references/protocol-buffers/) of the online docs. 54 | 55 | ## Contributing 56 | 57 | Generally, you should not need to modify this repository unless you 58 | are adding new client-facing features to Riak or fixing a 59 | bug. Nevertheless, we encourage contributions to `riak_pb` from the 60 | community. 61 | 62 | 1. Fork the [`riak_pb`](https://github.com/basho/riak_pb) repository 63 | on Github. 64 | 2. Clone your fork or add the remote if you already have a clone of 65 | the repository. 66 | 67 | ``` 68 | git clone git@github.com:yourusername/riak_pb.git 69 | # or 70 | git remote add mine git@github.com:yourusername/riak_pb.git 71 | ``` 72 | 73 | 3. Create a topic branch for your change. 74 | 75 | ``` 76 | git checkout -b some-topic-branch 77 | ``` 78 | 79 | 4. Make your change and commit. Use a clear and descriptive commit 80 | message, spanning multiple lines if detailed explanation is needed. 81 | 5. Push to your fork of the repository and then send a pull-request 82 | through Github. 83 | 84 | ``` 85 | git push mine some-topic-branch 86 | ``` 87 | 88 | 6. A Basho engineer or community maintainer will review your patch and 89 | merge it into the main repository or send you feedback. 90 | 91 | ## Build Prerequisites 92 | 93 | * protoc v 2.5.0 94 | 95 | On OSX the default version installed by brew is 2.6.x (Nov 2015) 96 | this version is too new for the Maven protocol-buffers plugin. 97 | 98 | You can install the correct version like so. 99 | 100 | ``` 101 | brew tap homebrew/versions 102 | brew install homebrew/versions/protobuf250 103 | ``` 104 | 105 | -------------------------------------------------------------------------------- /RELNOTES.md: -------------------------------------------------------------------------------- 1 | Release Notes 2 | ============= 3 | * `2.2.1.0` 4 | * [add gsets support](https://github.com/basho/riak_pb/pull/229) 5 | * Update `erlang_protobuffs` to `0.9.1` 6 | * Update `erlang_hamcrest` to `0.4.2` 7 | * `2.1.4.2` 8 | * Update `erlang_protobuffs` to `0.9.0`. Supports quoting reserved words in a `R15`-compatible way. 9 | * [`2.1.4.1`](https://github.com/basho/riak_pb/issues?q=milestone%3Ariak_pb-2.1.4.1) 10 | * OTP 19 support in `riak_pb` and dependencies. 11 | -------------------------------------------------------------------------------- /build/publish: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -o errexit 4 | set -o nounset 5 | 6 | declare -r debug='false' 7 | declare -r tmpfile_file="/tmp/publish.$$.tmpfiles" 8 | 9 | function make_temp_file 10 | { 11 | local template="${1:-publish.$$.XXXXXX}" 12 | if [[ $template != *XXXXXX ]] 13 | then 14 | template="$template.XXXXXX" 15 | fi 16 | local tmp=$(mktemp -t "$template") 17 | echo "$tmp" >> "$tmpfile_file" 18 | echo "$tmp" 19 | } 20 | 21 | function now 22 | { 23 | date '+%Y-%m-%d %H:%M:%S' 24 | } 25 | 26 | function pwarn 27 | { 28 | echo "$(now) [warning]: $@" 1>&2 29 | } 30 | 31 | function perr 32 | { 33 | echo "$(now) [error]: $@" 1>&2 34 | } 35 | 36 | function pinfo 37 | { 38 | echo "$(now) [info]: $@" 39 | } 40 | 41 | function pdebug 42 | { 43 | if [[ $debug == 'true' ]] 44 | then 45 | echo "$(now) [debug]: $@" 46 | fi 47 | } 48 | 49 | function errexit 50 | { 51 | perr "$@" 52 | exit 1 53 | } 54 | 55 | function onexit 56 | { 57 | if [[ -f $tmpfile_file ]] 58 | then 59 | for tmpfile in $(< $tmpfile_file) 60 | do 61 | pdebug "removing temp file $tmpfile" 62 | rm -f $tmpfile 63 | done 64 | rm -f $tmpfile_file 65 | fi 66 | } 67 | 68 | function gh_publish { 69 | if [[ -z $version_string ]] 70 | then 71 | errexit 'gh_publish: version_string required' 72 | fi 73 | 74 | # NB: we use a X.Y.Z.N tag 75 | local -r release_json="{ 76 | \"tag_name\" : \"$version_string\", 77 | \"name\" : \"Riak PB $version_string\", 78 | \"body\" : \"riak_pb $version_string\nhttps://github.com/basho/riak_pb/blob/master/RELNOTES.md\", 79 | \"draft\" : false, 80 | \"prerelease\" : $is_prerelease 81 | }" 82 | 83 | pdebug "Release JSON: $release_json" 84 | 85 | local curl_content_file="$(make_temp_file)" 86 | local curl_stdout_file="$(make_temp_file)" 87 | local curl_stderr_file="$(make_temp_file)" 88 | 89 | curl -4so $curl_content_file -w '%{http_code}' -XPOST \ 90 | -H "Authorization: token $(< $github_api_key_file)" -H 'Content-type: application/json' \ 91 | 'https://api.github.com/repos/basho/riak_pb/releases' -d "$release_json" 1> "$curl_stdout_file" 2> "$curl_stderr_file" 92 | if [[ $? != 0 ]] 93 | then 94 | errexit "curl error exited with code: '$?' see '$curl_stderr_file'" 95 | fi 96 | 97 | local -i curl_rslt="$(< $curl_stdout_file)" 98 | if (( curl_rslt == 422 )) 99 | then 100 | pwarn "Release in GitHub already exists! (http code: '$curl_rslt')" 101 | curl -4so $curl_content_file -w '%{http_code}' -XGET \ 102 | -H "Authorization: token $(< $github_api_key_file)" -H 'Content-type: application/json' \ 103 | "https://api.github.com/repos/basho/riak_pb/releases/tags/$version_string" 1> "$curl_stdout_file" 2> "$curl_stderr_file" 104 | if [[ $? != 0 ]] 105 | then 106 | errexit "curl error exited with code: '$?' see '$curl_stderr_file'" 107 | fi 108 | elif (( curl_rslt != 201 )) 109 | then 110 | errexit "Creating release in GitHub failed with http code '$curl_rslt'" 111 | fi 112 | } 113 | 114 | trap onexit EXIT 115 | 116 | declare -r version_string="${1:-unknown}" 117 | 118 | if [[ ! $version_string =~ ^[0-9].[0-9].[0-9](.[0-9])?(-[a-z]+[0-9]+)?$ ]] 119 | then 120 | errexit 'first argument must be valid version string in X.Y.Z.N format' 121 | fi 122 | 123 | is_prerelease='false' 124 | if [[ $version_string =~ ^[0-9].[0-9].[0-9](.[0-9])?-[a-z]+[0-9]+$ ]] 125 | then 126 | pinfo "publishing pre-release version: $version_string" 127 | is_prerelease='true' 128 | else 129 | pinfo "publishing version $version_string" 130 | fi 131 | 132 | declare -r current_branch="$(git rev-parse --abbrev-ref HEAD)" 133 | 134 | if [[ $debug == 'false' && $is_prerelease == 'false' && $current_branch != 'master' ]] 135 | then 136 | errexit 'publish must be run on master branch' 137 | fi 138 | 139 | declare -r github_api_key_file="$HOME/.ghapi" 140 | if [[ ! -s $github_api_key_file ]] 141 | then 142 | errexit "please save your GitHub API token in $github_api_key_file" 143 | fi 144 | 145 | # Validate commands 146 | if ! hash curl 2>/dev/null 147 | then 148 | errexit "'curl' must be in your PATH" 149 | fi 150 | 151 | validate=${2:-''} 152 | if [[ $validate == 'validate' ]] 153 | then 154 | exit 0 155 | fi 156 | 157 | gh_publish 158 | -------------------------------------------------------------------------------- /eqc/bucket_props_codec_eqc.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2013 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | -module(bucket_props_codec_eqc). 21 | -ifdef(EQC). 22 | -include_lib("eqc/include/eqc.hrl"). 23 | -include_lib("eunit/include/eunit.hrl"). 24 | 25 | -compile([export_all, nowarn_export_all]). 26 | 27 | -define(QC_OUT(P), eqc:on_output(fun(F,TL) -> 28 | io:format(user, F, TL) 29 | end, P)). 30 | 31 | %%==================================================================== 32 | %% Eunit integration 33 | %%==================================================================== 34 | bucket_codec_test_() -> 35 | [{timeout, 10, 36 | ?_test(begin 37 | eqc:quickcheck(?QC_OUT(eqc:testing_time(4, prop_codec()))) 38 | end)}]. 39 | 40 | %%==================================================================== 41 | %% Properties 42 | %%==================================================================== 43 | prop_codec() -> 44 | ?FORALL(Props, sortuniq(list(bucket_prop())), 45 | ?WHENFAIL(begin 46 | io:format("Props: ~p~n~n", [Props]) 47 | end, 48 | begin 49 | Props2 = riak_pb_codec:decode_bucket_props( 50 | riak_pb:decode_msg( 51 | iolist_to_binary(riak_pb:encode_msg( 52 | riak_pb_codec:encode_bucket_props(Props))), 53 | rpbbucketprops)), 54 | Props =:= lists:sort(Props2) 55 | end)). 56 | 57 | %%==================================================================== 58 | %% Shell helpers 59 | %%==================================================================== 60 | qc() -> 61 | qc(2000). 62 | 63 | qc(NumTests) -> 64 | quickcheck(numtests(NumTests, prop_codec())). 65 | 66 | check() -> 67 | eqc:check(prop_codec(), eqc:current_counterexample()). 68 | 69 | %%==================================================================== 70 | %% Generators 71 | %%==================================================================== 72 | bucket_prop() -> 73 | oneof([num(n_val), 74 | flag(allow_mult), 75 | flag(last_write_wins), 76 | commit(precommit), 77 | commit(postcommit), 78 | chash(), 79 | linkfun(), 80 | num(old_vclock), 81 | num(young_vclock), 82 | num(big_vclock), 83 | num(small_vclock), 84 | quorum(pr), 85 | quorum(r), 86 | quorum(w), 87 | quorum(pw), 88 | quorum(dw), 89 | quorum(rw), 90 | flag(basic_quorum), 91 | flag(notfound_ok), 92 | backend(), 93 | flag(search), 94 | repl(), 95 | yz_index(), 96 | datatype(), 97 | flag(consistent), 98 | flag(write_once)]). 99 | 100 | sortuniq(Gen) -> 101 | ?LET(L, Gen, lists:ukeysort(1,L)). 102 | 103 | flag(Prop) -> 104 | ?LET(B, bool(), {Prop, B}). 105 | 106 | num(Prop) -> 107 | ?LET(N, nat(), {Prop, N}). 108 | 109 | quorum(Prop) -> 110 | ?LET(V, oneof([one, quorum, all, default, nat()]), {Prop, V}). 111 | 112 | backend() -> 113 | ?LET(B, non_empty(binary()), {backend, B}). 114 | 115 | repl() -> 116 | ?LET(R, oneof([false, realtime, fullsync, true]), {repl, R}). 117 | 118 | commit(Prop) -> 119 | ?LET(C, non_empty(list(commit_hook())), {Prop, C}). 120 | 121 | commit_hook() -> 122 | ?LET(H, oneof([modfun_hook(), name_hook()]), {struct, H}). 123 | 124 | modfun_hook() -> 125 | ?LET({M,F}, {non_empty(binary()), non_empty(binary())}, 126 | [{<<"mod">>, M}, {<<"fun">>, F}]). 127 | 128 | name_hook() -> 129 | ?LET(N, non_empty(binary()), [{<<"name">>, N}]). 130 | 131 | chash() -> 132 | ?LET({M,F}, {atom(), atom()}, {chash_keyfun,{M,F}}). 133 | 134 | atom() -> 135 | ?LET(B, non_empty(binary()), binary_to_atom(B, latin1)). 136 | 137 | linkfun() -> 138 | ?LET({M,F}, {atom(), atom()}, {linkfun, {modfun, M, F}}). 139 | 140 | datatype() -> 141 | {datatype, elements([counter, set, map, hll])}. 142 | 143 | yz_index() -> 144 | {search_index, non_empty(binary())}. 145 | -endif. 146 | -------------------------------------------------------------------------------- /include/riak_pb_kv_codec.hrl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riak_pb_kv_codec.hrl: Riak KV PB codec header 4 | %% 5 | %% Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | %% Canonical names of riakc_obj and riak_object metadata fields 24 | -define(MD_CTYPE, <<"content-type">>). 25 | -define(MD_CHARSET, <<"charset">>). 26 | -define(MD_ENCODING, <<"content-encoding">>). 27 | -define(MD_VTAG, <<"X-Riak-VTag">>). 28 | -define(MD_LINKS, <<"Links">>). 29 | -define(MD_LASTMOD, <<"X-Riak-Last-Modified">>). 30 | -define(MD_USERMETA, <<"X-Riak-Meta">>). 31 | -define(MD_INDEX, <<"index">>). 32 | -define(MD_DELETED, <<"X-Riak-Deleted">>). 33 | 34 | %% Content-Type for Erlang term_to_binary format 35 | -define(CTYPE_ERLANG_BINARY, "application/x-erlang-binary"). 36 | 37 | %% Quorum value encodings 38 | -define(UINT_MAX, 16#ffffffff). 39 | -define(RIAKPB_RW_ONE, ?UINT_MAX-1). 40 | -define(RIAKPB_RW_QUORUM, ?UINT_MAX-2). 41 | -define(RIAKPB_RW_ALL, ?UINT_MAX-3). 42 | -define(RIAKPB_RW_DEFAULT, ?UINT_MAX-4). 43 | -------------------------------------------------------------------------------- /include/riak_ts_ttb.hrl: -------------------------------------------------------------------------------- 1 | -define(TTB_MSG_CODE, 104). 2 | -------------------------------------------------------------------------------- /rebar.config: -------------------------------------------------------------------------------- 1 | %% -*- mode: erlang -*- 2 | {minimum_otp_vsn, "22.0"}. 3 | 4 | {erl_opts, [ 5 | debug_info, 6 | warnings_as_errors, 7 | {platform_define, "^[0-9]+", namespaced_types}, 8 | {i, "./_build/default/plugins/gpb/include"} 9 | ]}. 10 | 11 | {xref_checks, [ 12 | undefined_function_calls, 13 | undefined_functions, 14 | deprecated_function_calls, 15 | deprecated_functions 16 | ]}. 17 | 18 | {profiles, [ 19 | {gha, [{erl_opts, [{d, 'GITHUBEXCLUDE'}]}]} 20 | ]}. 21 | 22 | {eunit_opts, [verbose]}. 23 | 24 | {plugins, [{rebar3_gpb_plugin, {git, "https://github.com/basho/rebar3_gpb_plugin", {tag, "2.15.1+riak.3.0.4"}}}, 25 | {riak_pb_msgcodegen, {git, "https://github.com/basho/riak_pb_msgcodegen", {tag, "1.0.0"}}}, 26 | {eqc_rebar, {git, "https://github.com/Quviq/eqc-rebar", {branch, "master"}}} 27 | ]}. 28 | 29 | {deps, []}. 30 | 31 | {gpb_opts, [{module_name_suffix, "_pb"}, 32 | {msg_name_to_lower, true}, 33 | {defaults_for_omitted_optionals, true}, 34 | {i, "src"}]}. 35 | 36 | {provider_hooks, [ 37 | {pre, [{compile, {protobuf, compile}}, 38 | {compile, riak_pb_msgcodegen}]} 39 | ]}. 40 | 41 | {edoc_opts, [{preprocess, true}]}. 42 | 43 | -------------------------------------------------------------------------------- /rebar3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/basho/riak_pb/7a5e535217c13a32f3041888b0d46e9b4476065c/rebar3 -------------------------------------------------------------------------------- /src/riak.proto: -------------------------------------------------------------------------------- 1 | /* ------------------------------------------------------------------- 2 | ** 3 | ** riak.proto: Protocol buffers for Riak 4 | ** 5 | ** Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved. 6 | ** 7 | ** This file is provided to you under the Apache License, 8 | ** Version 2.0 (the "License"); you may not use this file 9 | ** except in compliance with the License. You may obtain 10 | ** a copy of the License at 11 | ** 12 | ** http://www.apache.org/licenses/LICENSE-2.0 13 | ** 14 | ** Unless required by applicable law or agreed to in writing, 15 | ** software distributed under the License is distributed on an 16 | ** "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | ** KIND, either express or implied. See the License for the 18 | ** specific language governing permissions and limitations 19 | ** under the License. 20 | ** 21 | ** ------------------------------------------------------------------- 22 | */ 23 | 24 | /* 25 | ** Revision: 1.4 26 | */ 27 | 28 | // Java package specifiers 29 | option java_package = "com.basho.riak.protobuf"; 30 | option java_outer_classname = "RiakPB"; 31 | 32 | 33 | // Error response - may be generated for any Req 34 | message RpbErrorResp { 35 | required bytes errmsg = 1; 36 | required uint32 errcode = 2; 37 | } 38 | 39 | // Get server info request - no message defined, just send RpbGetServerInfoReq message code 40 | message RpbGetServerInfoResp { 41 | optional bytes node = 1; 42 | optional bytes server_version = 2; 43 | } 44 | 45 | // Key/value pair - used for user metadata, indexes, search doc fields 46 | message RpbPair { 47 | required bytes key = 1; 48 | optional bytes value = 2; 49 | } 50 | 51 | // Get bucket properties request 52 | message RpbGetBucketReq { 53 | required bytes bucket = 1; 54 | optional bytes type = 2; 55 | } 56 | 57 | // Get bucket properties response 58 | message RpbGetBucketResp { 59 | required RpbBucketProps props = 1; 60 | } 61 | 62 | // Set bucket properties request 63 | message RpbSetBucketReq { 64 | required bytes bucket = 1; 65 | required RpbBucketProps props = 2; 66 | optional bytes type = 3; 67 | } 68 | 69 | // Set bucket properties response - no message defined, just send 70 | // RpbSetBucketResp 71 | 72 | // Reset bucket properties request 73 | message RpbResetBucketReq { 74 | required bytes bucket = 1; 75 | optional bytes type = 2; 76 | } 77 | 78 | // Get bucket properties request 79 | message RpbGetBucketTypeReq { 80 | required bytes type = 1; 81 | } 82 | 83 | // Set bucket properties request 84 | message RpbSetBucketTypeReq { 85 | required bytes type = 1; 86 | required RpbBucketProps props = 2; 87 | } 88 | 89 | // Set bucket properties response - no message defined, just send 90 | // RpbSetBucketResp 91 | 92 | // Module-Function pairs for commit hooks and other bucket properties 93 | // that take functions 94 | message RpbModFun { 95 | required bytes module = 1; 96 | required bytes function = 2; 97 | } 98 | 99 | // A commit hook, which may either be a modfun or a JavaScript named 100 | // function 101 | message RpbCommitHook { 102 | optional RpbModFun modfun = 1; 103 | optional bytes name = 2; 104 | } 105 | 106 | // Bucket properties 107 | message RpbBucketProps { 108 | // Declared in riak_core_app 109 | optional uint32 n_val = 1; 110 | optional bool allow_mult = 2; 111 | optional bool last_write_wins = 3; 112 | repeated RpbCommitHook precommit = 4; 113 | optional bool has_precommit = 5 [default = false]; 114 | repeated RpbCommitHook postcommit = 6; 115 | optional bool has_postcommit = 7 [default = false]; 116 | optional RpbModFun chash_keyfun = 8; 117 | 118 | // Declared in riak_kv_app 119 | optional RpbModFun linkfun = 9; 120 | optional uint32 old_vclock = 10; 121 | optional uint32 young_vclock = 11; 122 | optional uint32 big_vclock = 12; 123 | optional uint32 small_vclock = 13; 124 | optional uint32 pr = 14; 125 | optional uint32 r = 15; 126 | optional uint32 w = 16; 127 | optional uint32 pw = 17; 128 | optional uint32 dw = 18; 129 | optional uint32 rw = 19; 130 | optional bool basic_quorum = 20; 131 | optional bool notfound_ok = 21; 132 | 133 | // Used by riak_kv_multi_backend 134 | optional bytes backend = 22; 135 | 136 | // Used by riak_search bucket fixup 137 | optional bool search = 23; 138 | 139 | // Used by riak_repl bucket fixup 140 | enum RpbReplMode { 141 | FALSE = 0; 142 | REALTIME = 1; 143 | FULLSYNC = 2; 144 | TRUE = 3; 145 | } 146 | optional RpbReplMode repl = 24; 147 | 148 | // Search index 149 | optional bytes search_index = 25; 150 | 151 | // KV Datatypes 152 | optional bytes datatype = 26; 153 | 154 | // KV strong consistency 155 | optional bool consistent = 27; 156 | 157 | // KV fast path 158 | optional bool write_once = 28; 159 | 160 | // Hyperlolog DT Precision 161 | optional uint32 hll_precision = 29; 162 | } 163 | 164 | // Authentication request 165 | message RpbAuthReq { 166 | required bytes user = 1; 167 | required bytes password = 2; 168 | } 169 | -------------------------------------------------------------------------------- /src/riak_dt.proto: -------------------------------------------------------------------------------- 1 | /* ------------------------------------------------------------------- 2 | ** 3 | ** riak_dt.proto: Protocol buffers for Riak data structures/types 4 | ** 5 | ** Copyright (c) 2013 Basho Technologies, Inc. All Rights Reserved. 6 | ** 7 | ** This file is provided to you under the Apache License, 8 | ** Version 2.0 (the "License"); you may not use this file 9 | ** except in compliance with the License. You may obtain 10 | ** a copy of the License at 11 | ** 12 | ** http://www.apache.org/licenses/LICENSE-2.0 13 | ** 14 | ** Unless required by applicable law or agreed to in writing, 15 | ** software distributed under the License is distributed on an 16 | ** "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | ** KIND, either express or implied. See the License for the 18 | ** specific language governing permissions and limitations 19 | ** under the License. 20 | ** 21 | ** ------------------------------------------------------------------- 22 | */ 23 | 24 | /* 25 | ** Revision: 2.2.0 26 | */ 27 | 28 | // Java package specifiers 29 | option java_package = "com.basho.riak.protobuf"; 30 | option java_outer_classname = "RiakDtPB"; 31 | 32 | /* 33 | * =============== DATA STRUCTURES ================= 34 | */ 35 | 36 | /* 37 | * Field names in maps are composed of a binary identifier and a type. 38 | * This is so that two clients can create fields with the same name 39 | * but different types, and they converge independently. 40 | */ 41 | message MapField { 42 | /* 43 | * The types that can be stored in a map are limited to counters, 44 | * sets, registers, flags, and maps. 45 | */ 46 | enum MapFieldType { 47 | COUNTER = 1; 48 | SET = 2; 49 | REGISTER = 3; 50 | FLAG = 4; 51 | MAP = 5; 52 | } 53 | 54 | required bytes name = 1; 55 | required MapFieldType type = 2; 56 | } 57 | 58 | 59 | /* 60 | * An entry in a map is a pair of a field-name and value. The type 61 | * defined in the field determines which value type is expected. 62 | */ 63 | message MapEntry { 64 | required MapField field = 1; 65 | optional sint64 counter_value = 2; 66 | repeated bytes set_value = 3; 67 | optional bytes register_value = 4; 68 | optional bool flag_value = 5; 69 | repeated MapEntry map_value = 6; 70 | } 71 | 72 | /* 73 | * =============== FETCH ================= 74 | */ 75 | 76 | /* 77 | * The equivalent of KV's "RpbGetReq", results in a DtFetchResp. The 78 | * request-time options are limited to ones that are relevant to 79 | * structured data-types. 80 | */ 81 | message DtFetchReq { 82 | // The identifier: bucket, key and bucket-type 83 | required bytes bucket = 1; 84 | required bytes key = 2; 85 | required bytes type = 3; 86 | 87 | // Request options 88 | optional uint32 r = 4; 89 | optional uint32 pr = 5; 90 | optional bool basic_quorum = 6; 91 | optional bool notfound_ok = 7; 92 | optional uint32 timeout = 8; 93 | optional bool sloppy_quorum = 9; // Experimental, may change/disappear 94 | optional uint32 n_val = 10; // Experimental, may change/disappear 95 | 96 | // For read-only requests or context-free operations, you can set 97 | // this to false to reduce the size of the response payload. 98 | optional bool include_context = 11 [default=true]; 99 | 100 | optional uint32 node_confirms = 12; // Distinct phycical node quorum 101 | } 102 | 103 | 104 | /* 105 | * The value of the fetched data type. If present in the response, 106 | * then empty values (sets, maps) should be treated as such. 107 | */ 108 | message DtValue { 109 | optional sint64 counter_value = 1; 110 | repeated bytes set_value = 2; 111 | repeated MapEntry map_value = 3; 112 | /* We return an estimated cardinality of the Hyperloglog set 113 | * on fetch. 114 | */ 115 | optional uint64 hll_value = 4; 116 | repeated bytes gset_value = 5; 117 | } 118 | 119 | 120 | /* 121 | * The response to a "Fetch" request. If the `include_context` option 122 | * is specified, an opaque "context" value will be returned along with 123 | * the user-friendly data. When sending an "Update" request, the 124 | * client should send this context as well, similar to how one would 125 | * send a vclock for KV updates. The `type` field indicates which 126 | * value type to expect. When the `value` field is missing from the 127 | * message, the client should interpret it as a "not found". 128 | */ 129 | message DtFetchResp { 130 | enum DataType { 131 | COUNTER = 1; 132 | SET = 2; 133 | MAP = 3; 134 | HLL = 4; 135 | GSET = 5; 136 | } 137 | 138 | optional bytes context = 1; 139 | required DataType type = 2; 140 | optional DtValue value = 3; 141 | } 142 | 143 | /* 144 | * =============== UPDATE ================= 145 | */ 146 | 147 | /* 148 | * An operation to update a Counter, either on its own or inside a 149 | * Map. The `increment` field can be positive or negative. When absent, 150 | * the meaning is an increment by 1. 151 | */ 152 | message CounterOp { 153 | optional sint64 increment = 1; 154 | } 155 | 156 | /* 157 | * An operation to update a Set, either on its own or inside a Map. 158 | * Set members are opaque binary values, you can only add or remove 159 | * them from a Set. 160 | */ 161 | message SetOp { 162 | repeated bytes adds = 1; 163 | repeated bytes removes = 2; 164 | } 165 | 166 | /* 167 | * An operation to update a GSet, on its own. 168 | * GSet members are opaque binary values, you can only add 169 | * them to a Set. 170 | */ 171 | message GSetOp { 172 | repeated bytes adds = 1; 173 | } 174 | 175 | /* 176 | * An operation to update a Hyperloglog Set, a top-level DT. 177 | * You can only add to a HllSet. 178 | */ 179 | message HllOp { 180 | repeated bytes adds = 1; 181 | } 182 | 183 | /* 184 | * An operation to be applied to a value stored in a Map -- the 185 | * contents of an UPDATE operation. The operation field that is 186 | * present depends on the type of the field to which it is applied. 187 | */ 188 | message MapUpdate { 189 | /* 190 | * Flags only exist inside Maps and can only be enabled or 191 | * disabled, and there are no arguments to the operations. 192 | */ 193 | enum FlagOp { 194 | ENABLE = 1; 195 | DISABLE = 2; 196 | } 197 | 198 | required MapField field = 1; 199 | 200 | optional CounterOp counter_op = 2; 201 | optional SetOp set_op = 3; 202 | 203 | /* 204 | * There is only one operation on a register, which is to set its 205 | * value, therefore the "operation" is the new value. 206 | */ 207 | optional bytes register_op = 4; 208 | optional FlagOp flag_op = 5; 209 | optional MapOp map_op = 6; 210 | 211 | } 212 | 213 | /* 214 | * An operation to update a Map. All operations apply to individual 215 | * fields in the Map. 216 | */ 217 | message MapOp { 218 | /* 219 | * REMOVE removes a field and value from the Map. 220 | * UPDATE applies type-specific 221 | * operations to the values stored in the Map. 222 | */ 223 | repeated MapField removes = 1; 224 | repeated MapUpdate updates = 2; 225 | } 226 | 227 | /* 228 | * A "union" type for update operations. The included operation 229 | * depends on the datatype being updated. 230 | */ 231 | message DtOp { 232 | optional CounterOp counter_op = 1; 233 | optional SetOp set_op = 2; 234 | optional MapOp map_op = 3; 235 | /* Adding values to a hyperloglog (set) is just like adding values 236 | * to a set. 237 | */ 238 | optional HllOp hll_op = 4; 239 | optional GSetOp gset_op = 5; 240 | } 241 | 242 | /* 243 | * The equivalent of KV's "RpbPutReq", results in an empty response or 244 | * "DtUpdateResp" if `return_body` is specified, or the key is 245 | * assigned by the server. The request-time options are limited to 246 | * ones that are relevant to structured data-types. 247 | */ 248 | message DtUpdateReq { 249 | // The identifier 250 | required bytes bucket = 1; 251 | optional bytes key = 2; // missing key results in server-assigned key, like KV 252 | required bytes type = 3; // bucket type, not data-type (but the data-type is constrained per bucket-type) 253 | 254 | // Opaque update-context 255 | optional bytes context = 4; 256 | 257 | // The operations 258 | required DtOp op = 5; 259 | 260 | // Request options 261 | optional uint32 w = 6; 262 | optional uint32 dw = 7; 263 | optional uint32 pw = 8; 264 | optional bool return_body = 9 [default=false]; 265 | optional uint32 timeout = 10; 266 | optional bool sloppy_quorum = 11; // Experimental, may change/disappear 267 | optional uint32 n_val = 12; // Experimental, may change/disappear 268 | optional bool include_context = 13 [default=true]; // When return_body is true, should the context be returned too? 269 | optional uint32 node_confirms = 14; // Distinct phycical node quorum 270 | } 271 | 272 | 273 | /* 274 | * The equivalent of KV's "RpbPutResp", contains the assigned key if 275 | * it was assigned by the server, and the resulting value and context 276 | * if return_body was set. 277 | */ 278 | message DtUpdateResp { 279 | // The key, if assigned by the server 280 | optional bytes key = 1; 281 | 282 | // The opaque update context and value, if return_body was set. 283 | optional bytes context = 2; 284 | optional sint64 counter_value = 3; 285 | repeated bytes set_value = 4; 286 | repeated MapEntry map_value = 5; 287 | optional uint64 hll_value = 6; 288 | repeated bytes gset_value = 7; 289 | } 290 | -------------------------------------------------------------------------------- /src/riak_kv.proto: -------------------------------------------------------------------------------- 1 | /* ------------------------------------------------------------------- 2 | ** 3 | ** riak_kv.proto: Protocol buffers for riak KV 4 | ** 5 | ** Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. 6 | ** 7 | ** This file is provided to you under the Apache License, 8 | ** Version 2.0 (the "License"); you may not use this file 9 | ** except in compliance with the License. You may obtain 10 | ** a copy of the License at 11 | ** 12 | ** http://www.apache.org/licenses/LICENSE-2.0 13 | ** 14 | ** Unless required by applicable law or agreed to in writing, 15 | ** software distributed under the License is distributed on an 16 | ** "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | ** KIND, either express or implied. See the License for the 18 | ** specific language governing permissions and limitations 19 | ** under the License. 20 | ** 21 | ** ------------------------------------------------------------------- 22 | */ 23 | 24 | /* 25 | ** Revision: 1.4 26 | */ 27 | 28 | // Java package specifiers 29 | option java_package = "com.basho.riak.protobuf"; 30 | option java_outer_classname = "RiakKvPB"; 31 | 32 | import "riak.proto"; // for RpbPair 33 | 34 | // Get ClientId Request - no message defined, just send RpbGetClientIdReq message code 35 | message RpbGetClientIdResp { 36 | required bytes client_id = 1; // Client id in use for this connection 37 | } 38 | 39 | message RpbSetClientIdReq { 40 | required bytes client_id = 1; // Client id to use for this connection 41 | } 42 | // Set ClientId Request - no message defined, just send RpbSetClientIdReq message code 43 | 44 | 45 | // Get Request - retrieve bucket/key 46 | message RpbGetReq { 47 | required bytes bucket = 1; 48 | required bytes key = 2; 49 | optional uint32 r = 3; 50 | optional uint32 pr = 4; 51 | optional bool basic_quorum = 5; 52 | optional bool notfound_ok = 6; 53 | optional bytes if_modified = 7; // fail if the supplied vclock does not match 54 | optional bool head = 8; // return everything but the value 55 | optional bool deletedvclock = 9; // return the tombstone's vclock, if applicable 56 | optional uint32 timeout = 10; 57 | optional bool sloppy_quorum = 11; // Experimental, may change/disappear 58 | optional uint32 n_val = 12; // Experimental, may change/disappear 59 | optional bytes type = 13; // Bucket type, if not set we assume the 'default' type 60 | optional uint32 node_confirms = 14; // Distinct phycical node quorum 61 | } 62 | 63 | // Get Response - if the record was not found there will be no content/vclock 64 | message RpbGetResp { 65 | repeated RpbContent content = 1; 66 | optional bytes vclock = 2; // the opaque vector clock for the object 67 | optional bool unchanged = 3; 68 | } 69 | 70 | 71 | // Put request - if options.return_body is set then the updated metadata/data for 72 | // the key will be returned. 73 | message RpbPutReq { 74 | required bytes bucket = 1; 75 | optional bytes key = 2; 76 | optional bytes vclock = 3; 77 | required RpbContent content = 4; 78 | optional uint32 w = 5; 79 | optional uint32 dw = 6; 80 | optional bool return_body = 7; 81 | optional uint32 pw = 8; 82 | optional bool if_not_modified = 9; 83 | optional bool if_none_match = 10; 84 | optional bool return_head = 11; 85 | optional uint32 timeout = 12; 86 | optional bool asis = 13; 87 | optional bool sloppy_quorum = 14; // Experimental, may change/disappear 88 | optional uint32 n_val = 15; // Experimental, may change/disappear 89 | optional bytes type = 16; // Bucket type, if not set we assume the 'default' type 90 | optional uint32 node_confirms = 17; // Distinct phycical node quorum 91 | } 92 | 93 | // Put response - same as get response with optional key if one was generated 94 | message RpbPutResp { 95 | repeated RpbContent content = 1; 96 | optional bytes vclock = 2; // the opaque vector clock for the object 97 | optional bytes key = 3; // the key generated, if any 98 | } 99 | 100 | 101 | // Delete request 102 | message RpbDelReq { 103 | required bytes bucket = 1; 104 | required bytes key = 2; 105 | optional uint32 rw = 3; 106 | optional bytes vclock = 4; 107 | optional uint32 r = 5; 108 | optional uint32 w = 6; 109 | optional uint32 pr = 7; 110 | optional uint32 pw = 8; 111 | optional uint32 dw = 9; 112 | optional uint32 timeout = 10; 113 | optional bool sloppy_quorum = 11; // Experimental, may change/disappear 114 | optional uint32 n_val = 12; // Experimental, may change/disappear 115 | optional bytes type = 13; // Bucket type, if not set we assume the 'default' type 116 | } 117 | 118 | // Delete response - not defined, will return a RpbDelResp on success or RpbErrorResp on failure 119 | 120 | // List buckets request 121 | message RpbListBucketsReq { 122 | optional uint32 timeout = 1; 123 | optional bool stream = 2; 124 | optional bytes type = 3; // Bucket type, if not set we assume the 'default' type 125 | } 126 | 127 | // List buckets response - one or more of these packets will be sent 128 | // the last one will have done set true (and may not have any buckets in it) 129 | message RpbListBucketsResp { 130 | repeated bytes buckets = 1; 131 | optional bool done = 2; 132 | } 133 | 134 | 135 | // List keys in bucket request 136 | message RpbListKeysReq { 137 | required bytes bucket = 1; 138 | optional uint32 timeout = 2; 139 | optional bytes type = 3; // Bucket type, if not set we assume the 'default' type 140 | } 141 | 142 | // List keys in bucket response - one or more of these packets will be sent 143 | // the last one will have done set true (and may not have any keys in it) 144 | message RpbListKeysResp { 145 | repeated bytes keys = 1; 146 | optional bool done = 2; 147 | } 148 | 149 | 150 | // Map/Reduce request 151 | message RpbMapRedReq { 152 | required bytes request = 1; 153 | required bytes content_type = 2; 154 | } 155 | 156 | // Map/Reduce response 157 | // one or more of these packets will be sent the last one will have done set 158 | // true (and may not have phase/data in it) 159 | message RpbMapRedResp { 160 | optional uint32 phase = 1; 161 | optional bytes response = 2; 162 | optional bool done = 3; 163 | } 164 | 165 | // Secondary Index query request 166 | message RpbIndexReq { 167 | enum IndexQueryType { 168 | eq = 0; 169 | range = 1; 170 | } 171 | 172 | required bytes bucket = 1; 173 | required bytes index = 2; 174 | required IndexQueryType qtype = 3; 175 | optional bytes key = 4; // key here means equals value for index? 176 | optional bytes range_min = 5; 177 | optional bytes range_max = 6; 178 | optional bool return_terms = 7; 179 | optional bool stream = 8; 180 | optional uint32 max_results = 9; 181 | optional bytes continuation = 10; 182 | optional uint32 timeout = 11; 183 | optional bytes type = 12; // Bucket type, if not set we assume the 'default' type 184 | optional bytes term_regex = 13; 185 | // Whether to use pagination sort for non-paginated queries 186 | optional bool pagination_sort = 14; 187 | // parallel extraction extension 188 | optional bytes cover_context = 15; // chopped up coverage plan per-req 189 | optional bool return_body = 16; // Return values with keys, only works with $bucket/$key index queries 190 | 191 | } 192 | 193 | // Secondary Index query response 194 | message RpbIndexResp { 195 | repeated bytes keys = 1; 196 | repeated RpbPair results = 2; 197 | optional bytes continuation = 3; 198 | optional bool done = 4; 199 | } 200 | 201 | // Stolen from CS bucket response, to be used when return_body=true 202 | message RpbIndexBodyResp { 203 | repeated RpbIndexObject objects = 1; 204 | optional bytes continuation = 2; 205 | optional bool done = 3; 206 | } 207 | 208 | // added solely for riak_cs currently 209 | // for folding over a bucket and returning 210 | // objects. 211 | message RpbCSBucketReq { 212 | required bytes bucket = 1; 213 | required bytes start_key = 2; 214 | optional bytes end_key = 3; 215 | optional bool start_incl = 4 [default = true]; 216 | optional bool end_incl = 5 [default = false]; 217 | optional bytes continuation = 6; 218 | optional uint32 max_results = 7; 219 | optional uint32 timeout = 8; 220 | optional bytes type = 9; // Bucket type, if not set we assume the 'default' type 221 | // parallel extraction extension 222 | optional bytes cover_context = 10; // chopped up coverage plan per-req 223 | } 224 | 225 | // return for CS bucket fold 226 | message RpbCSBucketResp { 227 | repeated RpbIndexObject objects = 1; 228 | optional bytes continuation = 2; 229 | optional bool done = 3; 230 | } 231 | 232 | message RpbIndexObject { 233 | required bytes key = 1; 234 | required RpbGetResp object = 2; 235 | } 236 | 237 | // Content message included in get/put responses 238 | // Holds the value and associated metadata 239 | message RpbContent { 240 | required bytes value = 1; 241 | optional bytes content_type = 2; // the media type/format 242 | optional bytes charset = 3; 243 | optional bytes content_encoding = 4; 244 | optional bytes vtag = 5; 245 | repeated RpbLink links = 6; // links to other resources 246 | optional uint32 last_mod = 7; 247 | optional uint32 last_mod_usecs = 8; 248 | repeated RpbPair usermeta = 9; // user metadata stored with the object 249 | repeated RpbPair indexes = 10; // user metadata stored with the object 250 | optional bool deleted = 11; 251 | } 252 | 253 | // Link metadata 254 | message RpbLink { 255 | optional bytes bucket = 1; 256 | optional bytes key = 2; 257 | optional bytes tag = 3; 258 | } 259 | 260 | // Counter update request 261 | message RpbCounterUpdateReq { 262 | required bytes bucket = 1; 263 | required bytes key = 2; 264 | required sint64 amount = 3; 265 | optional uint32 w = 4; 266 | optional uint32 dw = 5; 267 | optional uint32 pw = 6; 268 | optional bool returnvalue = 7; 269 | optional uint32 node_confirms = 8; // Distinct phycical node quorum 270 | } 271 | 272 | // Counter update response? No message | error response 273 | message RpbCounterUpdateResp { 274 | optional sint64 value = 1; 275 | } 276 | 277 | // counter value 278 | message RpbCounterGetReq { 279 | required bytes bucket = 1; 280 | required bytes key = 2; 281 | optional uint32 r = 3; 282 | optional uint32 pr = 4; 283 | optional bool basic_quorum = 5; 284 | optional bool notfound_ok = 6; 285 | optional uint32 node_confirms = 7; // Distinct phycical node quorum 286 | } 287 | 288 | // Counter value response 289 | message RpbCounterGetResp { 290 | optional sint64 value = 1; 291 | } 292 | 293 | // Get bucket-key preflist request 294 | message RpbGetBucketKeyPreflistReq { 295 | required bytes bucket = 1; 296 | required bytes key = 2; 297 | optional bytes type = 3; 298 | } 299 | 300 | // Get bucket-key preflist response 301 | message RpbGetBucketKeyPreflistResp { 302 | repeated RpbBucketKeyPreflistItem preflist = 1; 303 | } 304 | 305 | // Preflist item 306 | message RpbBucketKeyPreflistItem { 307 | required int64 partition = 1; 308 | required bytes node = 2; 309 | required bool primary = 3; 310 | } 311 | 312 | 313 | // Request a segmented coverage plan for the specified bucket 314 | message RpbCoverageReq { 315 | optional bytes type = 1; // Bucket type, if not set we assume the 'default' type 316 | required bytes bucket = 2; 317 | optional uint32 min_partitions = 3; // If undefined, we build a normal coverage plan. If >, <<"fun">>, <<"name">>'. 73 | %% Note that "mod" and "fun" must be used together, and "name" cannot 74 | %% be used if the other two are present. 75 | -type commit_hook_field() :: binary(). 76 | 77 | %% @type commit_hook_property(). 78 | %% 79 | %% Bucket properties that are commit hooks have this format. 80 | -type commit_hook_property() :: [ {struct, [{commit_hook_field(), binary()}]} ]. 81 | 82 | %% @doc Create an iolist of msg code and encoded message. Replaces 83 | %% `riakc_pb:encode/1'. 84 | 85 | -spec encode(atom() | tuple()) -> iolist(). 86 | 87 | encode(Msg) when is_atom(Msg) -> 88 | encode_msg_no_body(msg_code(Msg), Msg); 89 | encode({Msg}) when is_atom(Msg) -> 90 | encode_msg_no_body(msg_code(Msg), Msg); 91 | encode(Msg) when is_tuple(Msg) -> 92 | MsgType = element(1, Msg), 93 | Encoder = encoder_for(MsgType), 94 | [msg_code(MsgType) | Encoder:encode_msg(Msg)]. 95 | 96 | %% ------------------------------------------------------------ 97 | %% Encode a message when no content body is present (message atom 98 | %% only). 99 | %% 100 | %% For PB messages, this simply encodes the message code, which serves 101 | %% to identify the encoded message on the other side of the socket 102 | %% connection. 103 | %% ------------------------------------------------------------ 104 | 105 | encode_msg_no_body(MsgCode, _Msg) -> 106 | [MsgCode]. %% I/O layer will convert this to binary 107 | 108 | %% @doc Convert a property list to an RpbBucketProps message 109 | %% @private 110 | post_decode(Msg=#tsgetreq{key=K}) -> 111 | Msg#tsgetreq{key=riak_pb_ts_codec:decode_cells(K)}; 112 | post_decode(Msg=#tsputreq{rows=R}) -> 113 | Msg#tsputreq{rows=riak_pb_ts_codec:decode_rows(R)}; 114 | post_decode(Msg) -> 115 | Msg. 116 | 117 | %% @doc Decode a protocol buffer message given its type - if no bytes 118 | %% return the atom for the message code. Replaces `riakc_pb:decode/2'. 119 | -spec decode(integer(), binary()) -> atom() | tuple(). 120 | decode(MsgCode, <<>>) -> 121 | msg_type(MsgCode); 122 | decode(MsgCode, MsgData) -> 123 | Decoder = decoder_for(MsgCode), 124 | Decoded = Decoder:decode_msg(MsgData, msg_type(MsgCode)), 125 | post_decode(Decoded). 126 | 127 | %% @doc Converts a message code into the symbolic message 128 | %% name. Replaces `riakc_pb:msg_type/1'. 129 | -spec msg_type(integer()) -> atom(). 130 | msg_type(Int) -> 131 | riak_pb_messages:msg_type(Int). 132 | 133 | %% @doc Converts a symbolic message name into a message code. Replaces 134 | %% `riakc_pb:msg_code/1'. 135 | -spec msg_code(atom()) -> integer(). 136 | msg_code(Atom) -> riak_pb_messages:msg_code(Atom). 137 | 138 | %% @doc Selects the appropriate decoder for a message code. 139 | -spec decoder_for(pos_integer()) -> module(). 140 | decoder_for(N) -> 141 | riak_pb_messages:decoder_for(N). 142 | 143 | %% @doc Selects the appropriate PB encoder for a given message name. 144 | -spec encoder_for(atom()) -> module(). 145 | encoder_for(M) -> 146 | decoder_for(msg_code(M)). 147 | 148 | %% @doc Convert a true/false, 1/0 etc to a true/false for protocol 149 | %% buffers bool. Replaces `riakc_pb:pbify_bool/1'. 150 | -spec encode_bool(boolean() | integer()) -> boolean(). 151 | encode_bool(true) -> 152 | true; 153 | encode_bool(false) -> 154 | false; 155 | encode_bool(0) -> true; 156 | encode_bool(N) when is_integer(N) -> false. 157 | 158 | %% @doc Convert a protocol buffers boolean to an Erlang 159 | %% boolean. Replaces `riakc_pb:erlify_bool/1'. 160 | -spec decode_bool(boolean() | integer()) -> boolean(). 161 | decode_bool(true) -> true; 162 | decode_bool(false) -> false; 163 | decode_bool(0) -> false; 164 | decode_bool(1) -> true. 165 | 166 | %% @doc Make sure an atom/string/binary is definitely a 167 | %% binary. Replaces `riakc_pb:to_binary/1'. 168 | -spec to_binary(atom() | string() | binary()) -> binary(). 169 | to_binary(A) when is_atom(A) -> 170 | atom_to_binary(A, latin1); 171 | to_binary(L) when is_list(L) -> 172 | list_to_binary(L); 173 | to_binary(B) when is_binary(B) -> 174 | B. 175 | 176 | %% @doc Converts an arbitrary type to a list for sending in a 177 | %% PB. Replaces `riakc_pb:any_to_list/1'. 178 | -spec to_list(list() | atom() | binary() | integer()) -> list(). 179 | to_list(V) when is_list(V) -> 180 | V; 181 | to_list(V) when is_atom(V) -> 182 | atom_to_list(V); 183 | to_list(V) when is_binary(V) -> 184 | binary_to_list(V); 185 | to_list(V) when is_integer(V) -> 186 | integer_to_list(V). 187 | 188 | %% @doc Convert {K,V} tuple to protocol buffers 189 | -spec encode_pair({Key::binary(), Value::any()}) -> #rpbpair{}. 190 | encode_pair({K,V}) -> 191 | #rpbpair{key = to_binary(K), value = to_binary(V)}. 192 | 193 | %% @doc Convert RpbPair PB message to erlang {K,V} tuple 194 | -spec decode_pair(#rpbpair{}) -> {binary(), binary()}. 195 | decode_pair(#rpbpair{key = K, value = V}) -> 196 | {K, V}. 197 | 198 | 199 | %% @doc Convert an RpbBucketProps message to a property list 200 | -spec decode_bucket_props(PBProps::#rpbbucketprops{} | undefined) -> [proplists:property()]. 201 | decode_bucket_props(undefined) -> 202 | []; 203 | decode_bucket_props(#rpbbucketprops{n_val=N, 204 | allow_mult=AM, 205 | last_write_wins=LWW, 206 | precommit=Pre, 207 | has_precommit=HasPre, 208 | postcommit=Post, 209 | has_postcommit=HasPost, 210 | chash_keyfun=Chash, 211 | linkfun=Link, 212 | old_vclock=Old, 213 | young_vclock=Young, 214 | big_vclock=Big, 215 | small_vclock=Small, 216 | pr=PR, r=R, w=W, pw=PW, 217 | dw=DW, rw=RW, 218 | basic_quorum=BQ, 219 | notfound_ok=NFOK, 220 | backend=Backend, 221 | search=Search, 222 | repl=Repl, 223 | search_index=Index, 224 | datatype=Datatype, 225 | consistent=Consistent, 226 | write_once=WriteOnce, 227 | hll_precision=HllPrecision 228 | }) -> 229 | %% Extract numerical properties 230 | [ {P,V} || {P,V} <- [{n_val, N}, {old_vclock, Old}, {young_vclock, Young}, 231 | {big_vclock, Big}, {small_vclock, Small}, 232 | {hll_precision, HllPrecision}], 233 | V /= undefined ] ++ 234 | %% Extract booleans 235 | [ {BProp, decode_bool(Bool)} || 236 | {BProp, Bool} <- [{allow_mult, AM}, {last_write_wins, LWW}, 237 | {basic_quorum, BQ}, {notfound_ok, NFOK}, 238 | {search, Search}, {consistent, Consistent}, 239 | {write_once, WriteOnce}], 240 | Bool /= undefined ] ++ 241 | 242 | %% Extract commit hooks 243 | [ {PrePostProp, decode_commit_hooks(CList)} || 244 | {PrePostProp, CList, Included} <- [{precommit, Pre, HasPre}, {postcommit, Post, HasPost}], 245 | Included == true ] ++ 246 | 247 | %% Extract modfuns 248 | [ {MFProp, decode_modfun(MF, MFProp)} || {MFProp, MF} <- [{chash_keyfun, Chash}, 249 | {linkfun, Link}], 250 | MF /= undefined ] ++ 251 | 252 | %% Extract backend and Yokozuna index 253 | [ {P,V} || {P,V} <- [{backend, Backend}, {search_index, Index}], 254 | is_binary(V) ] ++ 255 | 256 | %% Extract quora 257 | [ {QProp, riak_pb_kv_codec:decode_quorum(Q)} || 258 | {QProp, Q} <- [{pr, PR}, {r, R}, {w, W}, {pw, PW}, {dw, DW}, {rw, RW}], 259 | Q /= undefined ] ++ 260 | 261 | %% Extract repl prop 262 | [ {repl, decode_repl(Repl)} || Repl /= undefined ] ++ 263 | 264 | %% Extract datatype prop 265 | [ {datatype, safe_to_atom(Datatype)} || is_binary(Datatype) ]. 266 | 267 | 268 | 269 | %% @doc Convert a property list to an RpbBucketProps message 270 | -spec encode_bucket_props([proplists:property()]) -> PBProps::#rpbbucketprops{}. 271 | encode_bucket_props(Props) -> 272 | encode_bucket_props(Props, #rpbbucketprops{}). 273 | 274 | %% @doc Convert a property list to an RpbBucketProps message 275 | %% @private 276 | -spec encode_bucket_props([proplists:property()], PBPropsIn::#rpbbucketprops{}) -> PBPropsOut::#rpbbucketprops{}. 277 | encode_bucket_props([], Pb) -> 278 | Pb; 279 | encode_bucket_props([{n_val, Nval} | Rest], Pb) -> 280 | encode_bucket_props(Rest, Pb#rpbbucketprops{n_val = Nval}); 281 | encode_bucket_props([{allow_mult, Flag} | Rest], Pb) -> 282 | encode_bucket_props(Rest, Pb#rpbbucketprops{allow_mult = encode_bool(Flag)}); 283 | encode_bucket_props([{last_write_wins, LWW}|Rest], Pb) -> 284 | encode_bucket_props(Rest, Pb#rpbbucketprops{last_write_wins = encode_bool(LWW)}); 285 | encode_bucket_props([{precommit, Precommit}|Rest], Pb) -> 286 | encode_bucket_props(Rest, Pb#rpbbucketprops{precommit = encode_commit_hooks(Precommit), 287 | has_precommit = true}); 288 | encode_bucket_props([{postcommit, Postcommit}|Rest], Pb) -> 289 | encode_bucket_props(Rest, Pb#rpbbucketprops{postcommit = encode_commit_hooks(Postcommit), 290 | has_postcommit = true}); 291 | encode_bucket_props([{chash_keyfun, ModFun}|Rest], Pb) -> 292 | encode_bucket_props(Rest, Pb#rpbbucketprops{chash_keyfun = encode_modfun(ModFun)}); 293 | encode_bucket_props([{linkfun, ModFun}|Rest], Pb) -> 294 | encode_bucket_props(Rest, Pb#rpbbucketprops{linkfun = encode_modfun(ModFun)}); 295 | encode_bucket_props([{old_vclock, Num}|Rest], Pb) -> 296 | encode_bucket_props(Rest, Pb#rpbbucketprops{old_vclock = Num}); 297 | encode_bucket_props([{young_vclock, Num}|Rest], Pb) -> 298 | encode_bucket_props(Rest, Pb#rpbbucketprops{young_vclock = Num}); 299 | encode_bucket_props([{big_vclock, Num}|Rest], Pb) -> 300 | encode_bucket_props(Rest, Pb#rpbbucketprops{big_vclock = Num}); 301 | encode_bucket_props([{small_vclock, Num}|Rest], Pb) -> 302 | encode_bucket_props(Rest, Pb#rpbbucketprops{small_vclock = Num}); 303 | encode_bucket_props([{pr, Q}|Rest], Pb) -> 304 | encode_bucket_props(Rest, Pb#rpbbucketprops{pr = riak_pb_kv_codec:encode_quorum(Q)}); 305 | encode_bucket_props([{r, Q}|Rest], Pb) -> 306 | encode_bucket_props(Rest, Pb#rpbbucketprops{r = riak_pb_kv_codec:encode_quorum(Q)}); 307 | encode_bucket_props([{w, Q}|Rest], Pb) -> 308 | encode_bucket_props(Rest, Pb#rpbbucketprops{w = riak_pb_kv_codec:encode_quorum(Q)}); 309 | encode_bucket_props([{pw, Q}|Rest], Pb) -> 310 | encode_bucket_props(Rest, Pb#rpbbucketprops{pw = riak_pb_kv_codec:encode_quorum(Q)}); 311 | encode_bucket_props([{dw, Q}|Rest], Pb) -> 312 | encode_bucket_props(Rest, Pb#rpbbucketprops{dw = riak_pb_kv_codec:encode_quorum(Q)}); 313 | encode_bucket_props([{rw, Q}|Rest], Pb) -> 314 | encode_bucket_props(Rest, Pb#rpbbucketprops{rw = riak_pb_kv_codec:encode_quorum(Q)}); 315 | encode_bucket_props([{basic_quorum, BQ}|Rest], Pb) -> 316 | encode_bucket_props(Rest, Pb#rpbbucketprops{basic_quorum = encode_bool(BQ)}); 317 | encode_bucket_props([{notfound_ok, NFOK}|Rest], Pb) -> 318 | encode_bucket_props(Rest, Pb#rpbbucketprops{notfound_ok = encode_bool(NFOK)}); 319 | encode_bucket_props([{backend, B}|Rest], Pb) -> 320 | encode_bucket_props(Rest, Pb#rpbbucketprops{backend = to_binary(B)}); 321 | encode_bucket_props([{search, S}|Rest], Pb) -> 322 | encode_bucket_props(Rest, Pb#rpbbucketprops{search = encode_bool(S)}); 323 | encode_bucket_props([{repl, Atom}|Rest], Pb) -> 324 | encode_bucket_props(Rest, Pb#rpbbucketprops{repl = encode_repl(Atom)}); 325 | encode_bucket_props([{search_index, B}|Rest], Pb) -> 326 | encode_bucket_props(Rest, Pb#rpbbucketprops{search_index = to_binary(B)}); 327 | encode_bucket_props([{datatype, D}|Rest], Pb) -> 328 | encode_bucket_props(Rest, Pb#rpbbucketprops{datatype = to_binary(D)}); 329 | encode_bucket_props([{consistent, S}|Rest], Pb) -> 330 | encode_bucket_props(Rest, Pb#rpbbucketprops{consistent = encode_bool(S)}); 331 | encode_bucket_props([{write_once, S}|Rest], Pb) -> 332 | encode_bucket_props(Rest, Pb#rpbbucketprops{write_once = encode_bool(S)}); 333 | encode_bucket_props([{hll_precision, Num}|Rest], Pb) -> 334 | encode_bucket_props(Rest, Pb#rpbbucketprops{hll_precision = Num}); 335 | encode_bucket_props([_Ignore|Rest], Pb) -> 336 | %% Ignore any properties not explicitly part of the PB message 337 | encode_bucket_props(Rest, Pb). 338 | 339 | %% @doc Converts a module-function specification into a RpbModFun message. 340 | -spec encode_modfun(modfun_property()) -> #rpbmodfun{}. 341 | encode_modfun({struct, Props}) -> 342 | {<<"mod">>, Mod} = lists:keyfind(<<"mod">>, 1, Props), 343 | {<<"fun">>, Fun} = lists:keyfind(<<"fun">>, 1, Props), 344 | encode_modfun({Mod, Fun}); 345 | encode_modfun({modfun, M, F}) -> 346 | encode_modfun({M, F}); 347 | encode_modfun({M, F}) -> 348 | #rpbmodfun{module=to_binary(M), function=to_binary(F)}. 349 | 350 | %% @doc Converts an RpbModFun message into the appropriate format for 351 | %% the given property. 352 | -spec decode_modfun(#rpbmodfun{}, atom()) -> modfun_property(). 353 | decode_modfun(MF, linkfun) -> 354 | {M,F} = decode_modfun(MF, undefined), 355 | {modfun, M, F}; 356 | decode_modfun(#rpbmodfun{module=Mod, function=Fun}, commit_hook) -> 357 | {struct, [{<<"mod">>, Mod}, {<<"fun">>, Fun}]}; 358 | decode_modfun(#rpbmodfun{module=Mod, function=Fun}=MF, _Prop) -> 359 | try 360 | {binary_to_existing_atom(Mod, latin1), binary_to_existing_atom(Fun, latin1)} 361 | catch 362 | error:badarg -> 363 | error_logger:warning_msg("Creating new atoms from protobuffs message! ~p", [MF]), 364 | {binary_to_atom(Mod, latin1), binary_to_atom(Fun, latin1)} 365 | end. 366 | 367 | %% @doc Converts a list of commit hooks into a list of RpbCommitHook 368 | %% messages. 369 | -spec encode_commit_hooks([commit_hook_property()]) -> [ #rpbcommithook{} ]. 370 | encode_commit_hooks(Hooks) -> 371 | [ encode_commit_hook(Hook) || Hook <- Hooks ]. 372 | 373 | encode_commit_hook({struct, Props}=Hook) -> 374 | FoundProps = [ lists:keymember(Field, 1, Props) || 375 | Field <- [<<"mod">>, <<"fun">>, <<"name">>]], 376 | case FoundProps of 377 | [true, true, _] -> 378 | #rpbcommithook{modfun=encode_modfun(Hook)}; 379 | [false, false, true] -> 380 | {<<"name">>, Name} = lists:keyfind(<<"name">>, 1, Props), 381 | #rpbcommithook{name=to_binary(Name)}; 382 | _ -> 383 | erlang:error(badarg, [Hook]) 384 | end. 385 | 386 | %% @doc Converts a list of RpbCommitHook messages into commit hooks. 387 | -spec decode_commit_hooks([ #rpbcommithook{} ]) -> 388 | [ commit_hook_property() | modfun_property()]. 389 | decode_commit_hooks(Hooks) -> 390 | [ decode_commit_hook(Hook) || Hook <- Hooks, 391 | Hook =/= #rpbcommithook{modfun=undefined, name=undefined} ]. 392 | 393 | decode_commit_hook(#rpbcommithook{modfun = Modfun}) when Modfun =/= undefined -> 394 | decode_modfun(Modfun, commit_hook); 395 | decode_commit_hook(#rpbcommithook{name = Name}) when Name =/= undefined -> 396 | {struct, [{<<"name">>, Name}]}. 397 | 398 | encode_repl(Bin) when is_binary(Bin) -> encode_repl(binary_to_existing_atom(Bin, latin1)); 399 | encode_repl(both) -> 'TRUE'; 400 | encode_repl(true) -> 'TRUE'; 401 | encode_repl(false) -> 'FALSE'; 402 | encode_repl(realtime) -> 'REALTIME'; 403 | encode_repl(fullsync) -> 'FULLSYNC'. 404 | 405 | decode_repl('TRUE') -> true; 406 | decode_repl('FALSE') -> false; 407 | decode_repl('REALTIME') -> realtime; 408 | decode_repl('FULLSYNC') -> fullsync. 409 | 410 | safe_to_atom(Binary) when is_binary(Binary) -> 411 | try 412 | binary_to_existing_atom(Binary, latin1) 413 | catch 414 | error:badarg -> 415 | error_logger:warning_msg("Creating new atom from protobuffs message! ~p", [Binary]), 416 | binary_to_atom(Binary, latin1) 417 | end. 418 | 419 | -ifdef(TEST). 420 | -include("riak_kv_pb.hrl"). 421 | -include("riak_dt_pb.hrl"). 422 | 423 | %% One necessary omission: we do not have any messages today that 424 | %% include functions, so we cannot test decoding such records. 425 | 426 | decode_eq(Message, <>, DecodeFun) -> 427 | ?assertEqual(Message, DecodeFun(MsgCode, Rest)); 428 | decode_eq(Message, IoList, DecodeFun) -> 429 | decode_eq(Message, iolist_to_binary(IoList), DecodeFun). 430 | 431 | record_test() -> 432 | Req = 433 | #rpbgetreq{n_val=4, 434 | notfound_ok=true, 435 | bucket = <<"bucket">>, 436 | key = <<"key">>}, 437 | decode_eq(Req, encode(Req), fun decode/2). 438 | 439 | optional_booleans_test() -> 440 | Req = #dtfetchreq{bucket = "bucket", 441 | key = <<"key">>, 442 | type = <<"type">>}, 443 | ?assertEqual(undefined, Req#dtfetchreq.r), 444 | ?assertEqual(undefined, Req#dtfetchreq.pr), 445 | ?assertEqual(undefined, Req#dtfetchreq.basic_quorum), 446 | ?assertEqual(undefined, Req#dtfetchreq.notfound_ok), 447 | ?assertEqual(undefined, Req#dtfetchreq.timeout), 448 | ?assertEqual(undefined, Req#dtfetchreq.sloppy_quorum), 449 | ?assertEqual(undefined, Req#dtfetchreq.n_val), 450 | ?assertEqual(true, Req#dtfetchreq.include_context), 451 | DecodedReq = #dtfetchreq{bucket = <<"bucket">>, 452 | key = <<"key">>, 453 | type = <<"type">>, 454 | r = undefined, 455 | pr = undefined, 456 | basic_quorum = undefined, 457 | notfound_ok = undefined, 458 | timeout = undefined, 459 | sloppy_quorum = undefined, 460 | n_val = undefined, 461 | include_context = true}, 462 | decode_eq(DecodedReq, encode(Req), fun decode/2). 463 | 464 | empty_atoms_test() -> 465 | %% Empty messages are either empty records or atoms, depending on 466 | %% whether the .proto file defines the message as an empty record 467 | %% or ignores it. On the receiving end they are all atoms. 468 | 469 | Resp = tsdelresp, %% .proto defines as empty record 470 | 471 | decode_eq(Resp, encode(Resp), fun decode/2). 472 | 473 | mixed_strings_test() -> 474 | %% Because the network layer will invoke iolist_to_binary/1 or its 475 | %% equivalent, on the sending side we can get away with using 476 | %% strings instead of binaries in records that expect the latter 477 | Req = 478 | #rpbgetreq{n_val=4, 479 | notfound_ok=true, 480 | bucket = "bucket", 481 | key = <<"key">>}, 482 | 483 | DecodedReq = 484 | #rpbgetreq{n_val=4, 485 | notfound_ok=true, 486 | bucket = <<"bucket">>, 487 | key = <<"key">>}, 488 | 489 | decode_eq(DecodedReq, encode(Req), fun decode/2). 490 | 491 | -endif. %% TEST 492 | -------------------------------------------------------------------------------- /src/riak_pb_dt_codec.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riak_pb_dt_codec: Protocol Buffers utility functions for Riak DT types 4 | %% 5 | %% Copyright (c) 2013 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | -module(riak_pb_dt_codec). 23 | 24 | -include("riak_dt_pb.hrl"). 25 | 26 | -include_lib("eunit/include/eunit.hrl"). 27 | 28 | -export([ 29 | encode_fetch_request/2, 30 | encode_fetch_request/3, 31 | decode_fetch_response/1, 32 | encode_fetch_response/3, 33 | encode_fetch_response/4, 34 | encode_update_request/3, 35 | encode_update_request/4, 36 | decode_operation/1, 37 | decode_operation/2, 38 | operation_type/1, 39 | decode_update_response/3, 40 | encode_update_response/4, 41 | encode_update_response/5, 42 | encode_operation/2 43 | ]). 44 | 45 | -import(riak_pb_kv_codec, [encode_quorum/1]). 46 | 47 | -export_type([context/0]). 48 | 49 | %% Value types 50 | -type context() :: binary() | undefined. 51 | -type counter_value() :: integer(). 52 | -type set_value() :: [ binary() ]. 53 | -type hll_value() :: number(). 54 | -type gset_value() :: [ binary() ]. 55 | -type register_value() :: binary(). 56 | -type flag_value() :: boolean(). 57 | -type map_entry() :: {map_field(), embedded_value()}. 58 | -type map_field() :: {binary(), embedded_type()}. 59 | -type map_value() :: [ map_entry() ]. 60 | -type embedded_value() :: counter_value() | set_value() | register_value() 61 | | flag_value() | map_value(). 62 | -type toplevel_value() :: counter_value() | gset_value() | set_value() | map_value() 63 | | hll_value() | undefined. 64 | -type fetch_response() :: {toplevel_type(), toplevel_value(), context()}. 65 | 66 | %% Type names as atoms 67 | -type embedded_type() :: counter | set | register | flag | map. 68 | -type toplevel_type() :: counter | gset | set | map | hll. 69 | -type all_type() :: toplevel_type() | embedded_type(). 70 | 71 | %% Operations 72 | -type counter_op() :: increment | decrement | {increment | decrement, integer()}. 73 | -type simple_set_op() :: {add, binary()} | {remove, binary()} | {add_all, [binary()]} | {remove_all, [binary()]}. 74 | -type set_op() :: simple_set_op() | {update, [simple_set_op()]}. 75 | -type hll_op() :: {add, binary()} | {add_all, [binary()]}. 76 | -type simple_gset_op() :: {add, binary()} | {add_all, [binary()]}. 77 | -type gset_op() :: simple_gset_op(). 78 | -type flag_op() :: enable | disable. 79 | -type register_op() :: {assign, binary()}. 80 | -type simple_map_op() :: {remove, map_field()} | {update, map_field(), embedded_type_op()}. 81 | -type map_op() :: simple_map_op() | {update, [simple_map_op()]}. 82 | -type embedded_type_op() :: counter_op() | set_op() | register_op() | flag_op() | map_op(). 83 | -type toplevel_op() :: counter_op() | gset_op() | set_op() | map_op() | hll_op(). 84 | -type update() :: {toplevel_type(), toplevel_op(), context()}. 85 | 86 | %% Request options 87 | -type quorum() :: riak_pb_kv_codec:quorum(). 88 | -type update_opt() :: {w, quorum()} | {dw, quorum()} | {pw, quorum()} | 89 | return_body | {return_body, boolean()} | 90 | {timeout, pos_integer()} | 91 | sloppy_quorum | {sloppy_quorum, boolean()} | 92 | {n_val, pos_integer()}. 93 | -type fetch_opt() :: {r, quorum()} | {pr, quorum()} | 94 | basic_quorum | {basic_quorum, boolean()} | 95 | notfound_ok | {notfound_ok, boolean()} | 96 | {timeout, pos_integer()} | 97 | sloppy_quorum | {sloppy_quorum, boolean()} | 98 | {n_val, pos_integer()} | 99 | include_context | {include_context, boolean()}. 100 | 101 | %% Server-side type<->module mappings 102 | -type type_mappings() :: [{all_type(), module()}]. 103 | 104 | 105 | %% ========================= 106 | %% DATA STRUCTURES AND TYPES 107 | %% ========================= 108 | 109 | %% @doc Decodes a MapField message into a tuple of name and type. 110 | -spec decode_map_field(#mapfield{}, type_mappings()) -> map_field(). 111 | decode_map_field(#mapfield{name=Name,type=Type}, Mods) -> 112 | {Name, decode_type(Type, Mods)}. 113 | 114 | %% @doc Encodes a tuple of name and type into a MapField message. 115 | -spec encode_map_field(map_field()) -> #mapfield{}. 116 | encode_map_field({Name, Type}) -> 117 | #mapfield{name=Name, type=encode_type(Type)}. 118 | 119 | %% @doc Decodes an MapEntry message into a tuple of field and value. 120 | -spec decode_map_entry(#mapentry{}) -> map_entry(). 121 | decode_map_entry(Entry) -> 122 | decode_map_entry(Entry, []). 123 | 124 | %% @doc Decodes an MapEntry message into a tuple of field and value, 125 | %% using the given type mappings. 126 | -spec decode_map_entry(#mapentry{}, type_mappings()) -> map_entry(). 127 | decode_map_entry(#mapentry{field=#mapfield{type='COUNTER'}=Field, counter_value=Val}, Mods) -> 128 | {decode_map_field(Field, Mods), Val}; 129 | decode_map_entry(#mapentry{field=#mapfield{type='SET'}=Field, set_value=Val}, Mods) -> 130 | {decode_map_field(Field, Mods), Val}; 131 | decode_map_entry(#mapentry{field=#mapfield{type='REGISTER'}=Field, register_value=Val}, Mods) -> 132 | {decode_map_field(Field, Mods), Val}; 133 | decode_map_entry(#mapentry{field=#mapfield{type='FLAG'}=Field, flag_value=Val}, Mods) -> 134 | {decode_map_field(Field, Mods), Val}; 135 | decode_map_entry(#mapentry{field=#mapfield{type='MAP'}=Field, map_value=Val}, Mods) -> 136 | {decode_map_field(Field, Mods), [ decode_map_entry(Entry, Mods) || Entry <- Val ]}. 137 | 138 | 139 | %% @doc Encodes a tuple of field and value into a MapEntry message. 140 | -spec encode_map_entry(map_entry(), type_mappings()) -> #mapentry{}. 141 | encode_map_entry({{Name, counter=Type}, Value}, _Mods) when is_integer(Value) -> 142 | #mapentry{field=encode_map_field({Name, Type}), counter_value=Value}; 143 | encode_map_entry({{Name, set=Type}, Value}, _Mods) when is_list(Value) -> 144 | #mapentry{field=encode_map_field({Name, Type}), set_value=Value}; 145 | encode_map_entry({{Name, register=Type}, Value}, _Mods) when is_binary(Value) -> 146 | #mapentry{field=encode_map_field({Name, Type}), register_value=Value}; 147 | encode_map_entry({{Name, register=Type}, undefined}, _Mods) -> 148 | #mapentry{field=encode_map_field({Name, Type})}; 149 | encode_map_entry({{Name, flag=Type}, Value}, _Mods) when is_atom(Value) -> 150 | #mapentry{field=encode_map_field({Name, Type}), flag_value=encode_flag_value(Value)}; 151 | encode_map_entry({{Name, map=Type}, Value}, Mods) when is_list(Value) -> 152 | #mapentry{field=encode_map_field({Name, Type}), 153 | map_value=[ encode_map_entry(Entry, Mods) || Entry <- Value ]}; 154 | encode_map_entry({{Name, Type}, Value}, Mods) -> 155 | %% We reach this clause if the type is not in the shortname yet, 156 | %% but is a module name. 157 | case lists:keyfind(Type, 2, Mods) of 158 | false -> 159 | %% If you don't have a mapping, we can't encode it. 160 | erlang:error(badarg, [{{Name,Type},Value}, Mods]); 161 | {AtomType, Type} -> 162 | encode_map_entry({{Name,AtomType}, Value}, Mods) 163 | end. 164 | 165 | %% @doc Decodes a PB message type name into a module name according to 166 | %% the passed mappings. 167 | -spec decode_type(atom(), type_mappings()) -> atom(). 168 | decode_type(PBType, Mods) -> 169 | AtomType = decode_type(PBType), 170 | proplists:get_value(AtomType, Mods, AtomType). 171 | 172 | %% @doc Decodes a PB message type name into an atom type name. 173 | -spec decode_type(atom()) -> all_type(). 174 | decode_type('COUNTER') -> counter; 175 | decode_type('SET') -> set; 176 | decode_type('HLL') -> hll; 177 | decode_type('GSET') -> gset; 178 | decode_type('REGISTER') -> register; 179 | decode_type('FLAG') -> flag; 180 | decode_type('MAP') -> map. 181 | 182 | 183 | %% @doc Encodes an atom type name into the PB message equivalent. 184 | -spec encode_type(all_type()) -> atom(). 185 | encode_type(counter) -> 'COUNTER'; 186 | encode_type(set) -> 'SET'; 187 | encode_type(hll) -> 'HLL'; 188 | encode_type(gset) -> 'GSET'; 189 | encode_type(register) -> 'REGISTER'; 190 | encode_type(flag) -> 'FLAG'; 191 | encode_type(map) -> 'MAP'. 192 | 193 | %% @doc Encodes a flag value into its PB message equivalent. 194 | encode_flag_value(on) -> true; 195 | encode_flag_value(off) -> false; 196 | encode_flag_value(Other) -> Other. 197 | 198 | 199 | %% ======================== 200 | %% FETCH REQUEST / RESPONSE 201 | %% ======================== 202 | 203 | %% @doc Encodes a fetch request into a DtFetch message. 204 | -spec encode_fetch_request({binary(), binary()}, binary()) -> #dtfetchreq{}. 205 | encode_fetch_request(BucketAndType, Key) -> 206 | encode_fetch_request(BucketAndType, Key, []). 207 | 208 | -spec encode_fetch_request({binary(), binary()}, binary(), [fetch_opt()]) -> #dtfetchreq{}. 209 | encode_fetch_request({BType,Bucket}, Key, Options) -> 210 | encode_fetch_options(#dtfetchreq{bucket=Bucket,key=Key,type=BType}, Options). 211 | 212 | %% @doc Encodes request-time fetch options onto the DtFetch message. 213 | %% @private 214 | -spec encode_fetch_options(#dtfetchreq{}, [fetch_opt()]) -> #dtfetchreq{}. 215 | encode_fetch_options(Fetch, []) -> 216 | Fetch; 217 | encode_fetch_options(Fetch, [{r,R}|Tail]) -> 218 | encode_fetch_options(Fetch#dtfetchreq{r=encode_quorum(R)},Tail); 219 | encode_fetch_options(Fetch, [{pr,PR}|Tail]) -> 220 | encode_fetch_options(Fetch#dtfetchreq{pr=encode_quorum(PR)},Tail); 221 | encode_fetch_options(Fetch, [basic_quorum|Tail]) -> 222 | encode_fetch_options(Fetch, [{basic_quorum, true}|Tail]); 223 | encode_fetch_options(Fetch, [{basic_quorum, BQ}|Tail]) -> 224 | encode_fetch_options(Fetch#dtfetchreq{basic_quorum=BQ},Tail); 225 | encode_fetch_options(Fetch, [{node_confirms, NodeConfirms}|Tail]) -> 226 | encode_fetch_options(Fetch#dtfetchreq{node_confirms=encode_quorum(NodeConfirms)}, Tail); 227 | encode_fetch_options(Fetch, [notfound_ok|Tail]) -> 228 | encode_fetch_options(Fetch, [{notfound_ok, true}|Tail]); 229 | encode_fetch_options(Fetch, [{notfound_ok, NOK}|Tail]) -> 230 | encode_fetch_options(Fetch#dtfetchreq{notfound_ok=NOK},Tail); 231 | encode_fetch_options(Fetch, [{timeout, TO}|Tail]) -> 232 | encode_fetch_options(Fetch#dtfetchreq{timeout=TO},Tail); 233 | encode_fetch_options(Fetch, [sloppy_quorum|Tail]) -> 234 | encode_fetch_options(Fetch, [{sloppy_quorum, true}|Tail]); 235 | encode_fetch_options(Fetch, [{sloppy_quorum, RB}|Tail]) -> 236 | encode_fetch_options(Fetch#dtfetchreq{sloppy_quorum=RB},Tail); 237 | encode_fetch_options(Fetch, [{n_val, N}|Tail]) -> 238 | encode_fetch_options(Fetch#dtfetchreq{n_val=N}, Tail); 239 | encode_fetch_options(Fetch, [include_context|Tail]) -> 240 | encode_fetch_options(Fetch, [{include_context, true}|Tail]); 241 | encode_fetch_options(Fetch, [{include_context, IC}|Tail]) -> 242 | encode_fetch_options(Fetch#dtfetchreq{include_context=IC},Tail); 243 | encode_fetch_options(Fetch, [_|Tail]) -> 244 | encode_fetch_options(Fetch, Tail). 245 | 246 | %% @doc Decodes a FetchResponse into tuple of type, value and context. 247 | -spec decode_fetch_response(#dtfetchresp{}) -> fetch_response() | {notfound, toplevel_type()}. 248 | decode_fetch_response(#dtfetchresp{type=T, value=undefined}) -> 249 | {notfound, decode_type(T)}; 250 | decode_fetch_response(#dtfetchresp{context=Context, type='COUNTER', 251 | value=#dtvalue{counter_value=Val}}) -> 252 | {counter, Val, Context}; 253 | decode_fetch_response(#dtfetchresp{context=Context, type='SET', 254 | value=#dtvalue{set_value=Val}}) -> 255 | {set, Val, Context}; 256 | decode_fetch_response(#dtfetchresp{context=Context, type='HLL', 257 | value=#dtvalue{hll_value=Val}}) -> 258 | {hll, Val, Context}; 259 | decode_fetch_response(#dtfetchresp{context=Context, type='GSET', 260 | value=#dtvalue{gset_value=Val}}) -> 261 | {gset, Val, Context}; 262 | decode_fetch_response(#dtfetchresp{context=Context, type='MAP', 263 | value=#dtvalue{map_value=Val}}) -> 264 | {map, [ decode_map_entry(Entry) || Entry <- Val ], Context}. 265 | 266 | %% @doc Encodes the result of a fetch request into a FetchResponse message. 267 | -spec encode_fetch_response(toplevel_type(), toplevel_value(), context()) -> #dtfetchresp{}. 268 | encode_fetch_response(Type, Value, Context) -> 269 | encode_fetch_response(Type, Value, Context, []). 270 | 271 | %% @doc Encodes the result of a fetch request into a FetchResponse message. 272 | -spec encode_fetch_response(toplevel_type(), toplevel_value(), context(), 273 | type_mappings()) -> #dtfetchresp{}. 274 | encode_fetch_response(Type, undefined, _Context, _Mods) -> 275 | #dtfetchresp{type=encode_type(Type)}; 276 | encode_fetch_response(Type, Value, Context, Mods) -> 277 | Response = #dtfetchresp{context=Context, type=encode_type(Type)}, 278 | case Type of 279 | counter -> 280 | Response#dtfetchresp{value=#dtvalue{counter_value=Value}}; 281 | set -> 282 | Response#dtfetchresp{value=#dtvalue{set_value=Value}}; 283 | hll -> 284 | Response#dtfetchresp{value=#dtvalue{hll_value=Value}}; 285 | gset -> 286 | Response#dtfetchresp{value=#dtvalue{gset_value=Value}}; 287 | map -> 288 | Response#dtfetchresp{value=#dtvalue{map_value=[encode_map_entry(Entry, Mods) || Entry <- Value]}} 289 | end. 290 | 291 | %% ========================= 292 | %% UPDATE REQUEST / RESPONSE 293 | %% ========================= 294 | 295 | %% @doc Decodes a CounterOp message into a counter operation. 296 | -spec decode_counter_op(#counterop{}) -> counter_op(). 297 | decode_counter_op(#counterop{increment=Int}) when is_integer(Int) -> 298 | {increment, Int}; 299 | decode_counter_op(#counterop{increment=undefined}) -> 300 | increment. 301 | 302 | %% @doc Encodes a counter operation into a CounterOp message. 303 | -spec encode_counter_op(counter_op()) -> #counterop{}. 304 | encode_counter_op({increment, Int}) when is_integer(Int) -> 305 | #counterop{increment=Int}; 306 | encode_counter_op(increment) -> 307 | #counterop{}; 308 | encode_counter_op(decrement) -> 309 | #counterop{increment=-1}; 310 | encode_counter_op({decrement, Int}) when is_integer(Int) -> 311 | #counterop{increment=(-Int)}. 312 | 313 | %% @doc Decodes a SetOp message into a set operation. 314 | -spec decode_set_op(#setop{}) -> set_op(). 315 | decode_set_op(#setop{adds=A, removes=[]}) -> 316 | {add_all, A}; 317 | decode_set_op(#setop{adds=[], removes=R}) -> 318 | {remove_all, R}; 319 | decode_set_op(#setop{adds=A, removes=R}) -> 320 | {update, [{add_all, A}, {remove_all, R}]}. 321 | 322 | %% @doc Encodes a set operation into a SetOp message. 323 | -spec encode_set_op(set_op()) -> #setop{}. 324 | encode_set_op({update, Ops}) when is_list(Ops) -> 325 | lists:foldr(fun encode_set_update/2, #setop{}, Ops); 326 | encode_set_op({C, _}=Op) when add == C; add_all == C; 327 | remove == C; remove_all == C-> 328 | encode_set_op({update, [Op]}). 329 | 330 | %% @doc Folds a set update into the SetOp message. 331 | -spec encode_set_update(simple_set_op(), #setop{}) -> #setop{}. 332 | encode_set_update({add, Member}, #setop{adds=A}=S) when is_binary(Member) -> 333 | S#setop{adds=[Member|A]}; 334 | encode_set_update({add_all, Members}, #setop{adds=A}=S) when is_list(Members) -> 335 | S#setop{adds=Members++A}; 336 | encode_set_update({remove, Member}, #setop{removes=R}=S) when is_binary(Member) -> 337 | S#setop{removes=[Member|R]}; 338 | encode_set_update({remove_all, Members}, #setop{removes=R}=S) when is_list(Members) -> 339 | S#setop{removes=Members++R}. 340 | 341 | 342 | %% @doc Decodes a GSetOp message into a gset operation. 343 | -spec decode_gset_op(#gsetop{}) -> gset_op(). 344 | decode_gset_op(#gsetop{adds=A}) -> 345 | {add_all, A}. 346 | 347 | %% @doc Encodes a gset operation into a SetOp message. 348 | -spec encode_gset_op(gset_op()|{update, [simple_gset_op()]}) -> #gsetop{}. 349 | encode_gset_op({update, Ops}) when is_list(Ops) -> 350 | lists:foldr(fun encode_gset_update/2, #gsetop{}, Ops); 351 | encode_gset_op({C, _}=Op) when add == C; add_all == C -> 352 | encode_gset_op({update, [Op]}). 353 | 354 | %% @doc Folds a set update into the SetOp message. 355 | -spec encode_gset_update(simple_gset_op(), #gsetop{}) -> #gsetop{}. 356 | encode_gset_update({add, Member}, #gsetop{adds=A}=S) when is_binary(Member) -> 357 | S#gsetop{adds=[Member|A]}; 358 | encode_gset_update({add_all, Members}, #gsetop{adds=A}=S) when is_list(Members) -> 359 | S#gsetop{adds=Members++A}. 360 | 361 | %% @doc Decodes a operation name from a PB message into an atom. 362 | -spec decode_flag_op(atom()) -> atom(). 363 | 364 | decode_flag_op('ENABLE') -> enable; 365 | decode_flag_op('DISABLE') -> disable. 366 | 367 | %% @doc Encodes an atom operation name into the PB message equivalent. 368 | -spec encode_flag_op(atom()) -> atom(). 369 | encode_flag_op(enable) -> 'ENABLE'; 370 | encode_flag_op(disable) -> 'DISABLE'. 371 | 372 | %% @doc Decodes a HllOp message into a hll operation. 373 | -spec decode_hll_op(#hllop{}) -> hll_op(). 374 | decode_hll_op(#hllop{adds=A}) -> 375 | {add_all, A}. 376 | 377 | %% @doc Encodes an hll(set) update into the HllOp message. 378 | -spec encode_hll_op(hll_op()) -> #hllop{}. 379 | encode_hll_op({add, Member}) when is_binary(Member) -> 380 | #hllop{adds=[Member]}; 381 | encode_hll_op({add_all, Members}) when is_list(Members) -> 382 | #hllop{adds=Members}. 383 | 384 | %% @doc Decodes a MapUpdate message into a map field operation. 385 | -spec decode_map_update(#mapupdate{}, type_mappings()) -> {map_field(), embedded_type_op()}. 386 | decode_map_update(#mapupdate{field=#mapfield{name=N, type='COUNTER'=Type}, counter_op=#counterop{}=Op}, Mods) -> 387 | COp = decode_counter_op(Op), 388 | FType = decode_type(Type, Mods), 389 | {{N, FType}, COp}; 390 | decode_map_update(#mapupdate{field=#mapfield{name=N, type='SET'=Type}, set_op=#setop{}=Op}, Mods) -> 391 | SOp = decode_set_op(Op), 392 | FType = decode_type(Type, Mods), 393 | {{N, FType}, SOp}; 394 | decode_map_update(#mapupdate{field=#mapfield{name=N, type='REGISTER'=Type}, register_op=Op}, Mods) -> 395 | FType = decode_type(Type, Mods), 396 | {{N, FType}, {assign, Op}}; 397 | decode_map_update(#mapupdate{field=#mapfield{name=N, type='FLAG'=Type}, flag_op=Op}, Mods) -> 398 | FOp = decode_flag_op(Op), 399 | FType = decode_type(Type, Mods), 400 | {{N, FType}, FOp}; 401 | decode_map_update(#mapupdate{field=#mapfield{name=N, type='MAP'=Type}, map_op=Op}, Mods) -> 402 | MOp = decode_map_op(Op, Mods), 403 | FType = decode_type(Type, Mods), 404 | {{N, FType}, MOp}. 405 | 406 | %% @doc Encodes a map field operation into a MapUpdate message. 407 | -spec encode_map_update(map_field(), embedded_type_op()) -> #mapupdate{}. 408 | encode_map_update({_Name, counter}=Key, Op) -> 409 | #mapupdate{field=encode_map_field(Key), counter_op=encode_counter_op(Op)}; 410 | encode_map_update({_Name, set}=Key, Op) -> 411 | #mapupdate{field=encode_map_field(Key), set_op=encode_set_op(Op)}; 412 | encode_map_update({_Name, register}=Key, {assign, Value}) -> 413 | #mapupdate{field=encode_map_field(Key), register_op=Value}; 414 | encode_map_update({_Name, flag}=Key, Op) -> 415 | #mapupdate{field=encode_map_field(Key), flag_op=encode_flag_op(Op)}; 416 | encode_map_update({_Name, map}=Key, Op) -> 417 | #mapupdate{field=encode_map_field(Key), map_op=encode_map_op(Op)}. 418 | 419 | %% @doc Encodes a map operation into a MapOp message. 420 | -spec encode_map_op(map_op()) -> #mapop{}. 421 | encode_map_op({update, Ops}) -> 422 | lists:foldr(fun encode_map_op_update/2, #mapop{}, Ops); 423 | encode_map_op({Op, _}=C) when add == Op; remove == Op -> 424 | encode_map_op({update, [C]}); 425 | encode_map_op({update, _Field, _Ops}=C) -> 426 | encode_map_op({update, [C]}). 427 | 428 | %% @doc Folds a map update into the MapOp message. 429 | -spec encode_map_op_update(simple_map_op(), #mapop{}) -> #mapop{}. 430 | encode_map_op_update({remove, F}, #mapop{removes=R}=M) -> 431 | M#mapop{removes=[encode_map_field(F)|R]}; 432 | encode_map_op_update({update, F, Ops}, #mapop{updates=U}=M) when is_list(Ops) -> 433 | Updates = [ encode_map_update(F, Op) || Op <- Ops ], 434 | M#mapop{updates=Updates ++ U}; 435 | encode_map_op_update({update, F, Op}, #mapop{updates=U}=M) -> 436 | M#mapop{updates=[encode_map_update(F, Op) | U]}. 437 | 438 | 439 | -spec decode_map_op(#mapop{}, type_mappings()) -> map_op(). 440 | decode_map_op(#mapop{removes=Removes, updates=Updates}, Mods) -> 441 | {update, 442 | [ {remove, decode_map_field(R, Mods)} || R <- Removes ] ++ 443 | [ begin 444 | {Field, Op} = decode_map_update(U, Mods), 445 | {update, Field, Op} 446 | end || U <- Updates ]}. 447 | 448 | %% @doc Decodes a DtOperation message into a datatype-specific operation. 449 | -spec decode_operation(#dtop{}) -> toplevel_op(). 450 | decode_operation(Op) -> 451 | decode_operation(Op, []). 452 | 453 | -spec decode_operation(#dtop{}, type_mappings()) -> toplevel_op(). 454 | decode_operation(#dtop{counter_op=#counterop{}=Op}, _) -> 455 | decode_counter_op(Op); 456 | decode_operation(#dtop{set_op=#setop{}=Op}, _) -> 457 | decode_set_op(Op); 458 | decode_operation(#dtop{hll_op=#hllop{}=Op}, _) -> 459 | decode_hll_op(Op); 460 | decode_operation(#dtop{gset_op=#gsetop{}=Op}, _) -> 461 | decode_gset_op(Op); 462 | decode_operation(#dtop{map_op=#mapop{}=Op}, Mods) -> 463 | decode_map_op(Op, Mods). 464 | 465 | %% @doc Encodes a datatype-specific operation into a DtOperation message. 466 | -spec encode_operation(toplevel_op(), toplevel_type()) -> #dtop{}. 467 | encode_operation(Op, counter) -> 468 | #dtop{counter_op=encode_counter_op(Op)}; 469 | encode_operation(Op, set) -> 470 | #dtop{set_op=encode_set_op(Op)}; 471 | encode_operation(Op, hll) -> 472 | #dtop{hll_op=encode_hll_op(Op)}; 473 | encode_operation(Op, gset) -> 474 | #dtop{gset_op=encode_gset_op(Op)}; 475 | encode_operation(Op, map) -> 476 | #dtop{map_op=encode_map_op(Op)}. 477 | 478 | %% @doc Returns the type that the DtOp message expects to be performed 479 | %% on. 480 | -spec operation_type(#dtop{}) -> toplevel_type(). 481 | operation_type(#dtop{counter_op=#counterop{}}) -> 482 | counter; 483 | operation_type(#dtop{set_op=#setop{}}) -> 484 | set; 485 | operation_type(#dtop{hll_op=#hllop{}}) -> 486 | hll; 487 | operation_type(#dtop{gset_op=#gsetop{}}) -> 488 | gset; 489 | operation_type(#dtop{map_op=#mapop{}}) -> 490 | map. 491 | 492 | %% @doc Encodes an update request into a DtUpdate message. 493 | -spec encode_update_request({binary(), binary()}, binary() | undefined, update()) -> #dtupdatereq{}. 494 | encode_update_request({_,_}=BucketAndType, Key, {_,_,_}=Update) -> 495 | encode_update_request(BucketAndType, Key, Update, []). 496 | 497 | -spec encode_update_request({binary(), binary()}, binary() | undefined, update(), [update_opt()]) -> #dtupdatereq{}. 498 | encode_update_request({BType, Bucket}, Key, {DType, Op, Context}, Options) -> 499 | Update = #dtupdatereq{bucket=Bucket, 500 | key=Key, 501 | type=BType, 502 | context=Context, 503 | op=encode_operation(Op, DType)}, 504 | encode_update_options(Update, Options). 505 | 506 | %% @doc Encodes request-time update options onto the DtUpdate message. 507 | %% @private 508 | -spec encode_update_options(#dtupdatereq{}, [proplists:property()]) -> #dtupdatereq{}. 509 | encode_update_options(Update, []) -> 510 | Update; 511 | encode_update_options(Update, [{w,W}|Tail]) -> 512 | encode_update_options(Update#dtupdatereq{w=encode_quorum(W)},Tail); 513 | encode_update_options(Update, [{dw,DW}|Tail]) -> 514 | encode_update_options(Update#dtupdatereq{dw=encode_quorum(DW)},Tail); 515 | encode_update_options(Update, [{pw,PW}|Tail]) -> 516 | encode_update_options(Update#dtupdatereq{pw=encode_quorum(PW)},Tail); 517 | encode_update_options(Update, [{node_confirms,NodeConfirms}|Tail]) -> 518 | encode_update_options(Update#dtupdatereq{node_confirms=encode_quorum(NodeConfirms)},Tail); 519 | encode_update_options(Update, [return_body|Tail]) -> 520 | encode_update_options(Update, [{return_body, true}|Tail]); 521 | encode_update_options(Update, [{return_body, RB}|Tail]) -> 522 | encode_update_options(Update#dtupdatereq{return_body=RB},Tail); 523 | encode_update_options(Update, [{timeout, TO}|Tail]) -> 524 | encode_update_options(Update#dtupdatereq{timeout=TO},Tail); 525 | encode_update_options(Update, [sloppy_quorum|Tail]) -> 526 | encode_update_options(Update, [{sloppy_quorum, true}|Tail]); 527 | encode_update_options(Update, [{sloppy_quorum, RB}|Tail]) -> 528 | encode_update_options(Update#dtupdatereq{sloppy_quorum=RB},Tail); 529 | encode_update_options(Update, [{n_val, N}|Tail]) -> 530 | encode_update_options(Update#dtupdatereq{n_val=N}, Tail); 531 | encode_update_options(Update, [include_context|Tail]) -> 532 | encode_update_options(Update, [{include_context, true}|Tail]); 533 | encode_update_options(Update, [{include_context, IC}|Tail]) -> 534 | encode_update_options(Update#dtupdatereq{include_context=IC},Tail); 535 | encode_update_options(Update, [_|Tail]) -> 536 | encode_update_options(Update, Tail). 537 | 538 | %% @doc Decodes a DtUpdateResp message into erlang values. 539 | -spec decode_update_response(#dtupdateresp{}, Type::toplevel_type(), ReturnBodyExpected::boolean()) -> 540 | ok | {ok, Key::binary()} | {Key::binary(), fetch_response()} | fetch_response(). 541 | decode_update_response(#dtupdateresp{key=K}, _, false) -> 542 | case K of 543 | undefined -> ok; 544 | _ -> {ok, K} 545 | end; 546 | decode_update_response(#dtupdateresp{counter_value=C, context=Ctx}=Resp, counter, true) -> 547 | maybe_wrap_key({counter, C, Ctx}, Resp); 548 | decode_update_response(#dtupdateresp{hll_value=Hll, context=Ctx}=Resp, hll, 549 | true) -> 550 | maybe_wrap_key({hll, Hll, Ctx}, Resp); 551 | decode_update_response(#dtupdateresp{set_value=S, context=Ctx}=Resp, set, true) -> 552 | maybe_wrap_key({set, S, Ctx}, Resp); 553 | decode_update_response(#dtupdateresp{map_value=M, context=Ctx}=Resp, map, true) -> 554 | maybe_wrap_key({map, [ decode_map_entry(F) || F <- M ], Ctx}, Resp). 555 | 556 | maybe_wrap_key(Term, #dtupdateresp{key=undefined}) -> Term; 557 | maybe_wrap_key(Term, #dtupdateresp{key=K}) -> {K, Term}. 558 | 559 | %% @doc Encodes an update response into a DtUpdateResp message. 560 | -spec encode_update_response(toplevel_type(), toplevel_value(), binary(), 561 | context()) -> #dtupdateresp{}. 562 | encode_update_response(Type, Value, Key, Context) -> 563 | encode_update_response(Type, Value, Key, Context, []). 564 | 565 | %% @doc Encodes an update response into a DtUpdateResp message. 566 | -spec encode_update_response(toplevel_type(), toplevel_value(), binary(), 567 | context(), type_mappings()) -> #dtupdateresp{}. 568 | encode_update_response(counter, Value, Key, Context, _Mods) -> 569 | #dtupdateresp{key=Key, context=Context, counter_value=Value}; 570 | encode_update_response(set, Value, Key, Context, _Mods) -> 571 | #dtupdateresp{key=Key, context=Context, set_value=Value}; 572 | encode_update_response(hll, Value, Key, Context, _Mods) -> 573 | #dtupdateresp{key=Key, context=Context, hll_value=Value}; 574 | encode_update_response(map, Value, Key, Context, Mods) when is_list(Value) -> 575 | #dtupdateresp{key=Key, context=Context, 576 | map_value=[encode_map_entry(Entry, Mods) 577 | || Value /= undefined, Entry <- Value]}. 578 | -------------------------------------------------------------------------------- /src/riak_pb_kv_codec.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riak_pb_kv_codec: protocol buffer utility functions for Riak KV messages 4 | %% 5 | %% Copyright (c) 2012 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | %% @doc Utility functions for decoding and encoding Protocol Buffers 24 | %% messages related to Riak KV. These are used inside the client and 25 | %% server code and do not normally need to be used in application 26 | %% code. 27 | 28 | -module(riak_pb_kv_codec). 29 | 30 | -include("riak_kv_pb.hrl"). 31 | -include("riak_pb_kv_codec.hrl"). 32 | 33 | -ifdef(TEST). 34 | -include_lib("eunit/include/eunit.hrl"). 35 | -endif. 36 | 37 | -export([encode_contents/1, %% riakc_pb:pbify_rpbcontents 38 | decode_contents/1, %% riakc_pb:erlify_rpbcontents 39 | encode_content/1, %% riakc_pb:pbify_rpbcontent 40 | decode_content/1, %% riakc_pb:erlify_rpbcontent 41 | encode_content_meta/3, %% riakc_pb:pbify_rpbcontent_entry 42 | decode_content_meta/3, 43 | encode_pair/1, %% riakc_pb:pbify_rpbpair 44 | encode_index_pair/1, 45 | decode_pair/1, %% riakc_pb:erlify_rpbpair 46 | encode_link/1, %% riakc_pb:pbify_rpblink 47 | decode_link/1, %% riakc_pb:erlify_rpblink 48 | encode_quorum/1, 49 | decode_quorum/1, %% riak_kv_pb_socket:normalize_rw_value 50 | encode_apl_ann/1 51 | ]). 52 | 53 | -export_type([quorum/0]). 54 | -type quorum() :: symbolic_quorum() | non_neg_integer(). 55 | -type symbolic_quorum() :: one | quorum | all | default. 56 | -type value() :: binary(). 57 | 58 | -ifdef(namespaced_types). 59 | -type metadata() :: dict:dict(binary(), binary()). 60 | -else. 61 | -type metadata() :: dict(). 62 | -endif. 63 | 64 | -type contents() :: [{metadata(), value()}]. 65 | 66 | %% @doc Annotated preflist type 67 | -type preflist_with_pnum_ann() :: [{{non_neg_integer(), node()}, primary|fallback}]. 68 | 69 | 70 | %% @doc Convert a list of object {MetaData,Value} pairs to protocol 71 | %% buffers messages. 72 | -spec encode_contents(contents()) -> [#rpbcontent{}]. 73 | encode_contents(List) -> 74 | [ encode_content(C) || C <- List ]. 75 | 76 | %% @doc Convert a metadata/value pair into an #rpbcontent{} record 77 | -spec encode_content({metadata(), value()}) -> #rpbcontent{}. 78 | encode_content({MetadataIn, ValueIn}=C) -> 79 | {Metadata, Value} = 80 | case is_binary(ValueIn) of 81 | true -> 82 | C; 83 | false -> 84 | %% If the riak object was created using 85 | %% the native erlang interface, it is possible 86 | %% for the value to consist of arbitrary terms. 87 | %% PBC needs to send a binary, so replace the content type 88 | %% to mark it as an erlang binary and encode 89 | %% the term as a binary. 90 | {dict:store(?MD_CTYPE, ?CTYPE_ERLANG_BINARY, MetadataIn), 91 | term_to_binary(ValueIn)} 92 | end, 93 | dict:fold(fun encode_content_meta/3, #rpbcontent{value = Value}, Metadata). 94 | 95 | %% @doc Convert the metadata dictionary entries to protocol buffers 96 | -spec encode_content_meta(MetadataKey::string(), any(), tuple()) -> tuple(). 97 | encode_content_meta(?MD_CTYPE, ContentType, PbContent) when is_list(ContentType) -> 98 | PbContent#rpbcontent{content_type = ContentType}; 99 | encode_content_meta(?MD_CHARSET, Charset, PbContent) when is_list(Charset) -> 100 | PbContent#rpbcontent{charset = Charset}; 101 | encode_content_meta(?MD_ENCODING, Encoding, PbContent) when is_list(Encoding) -> 102 | PbContent#rpbcontent{content_encoding = Encoding}; 103 | encode_content_meta(?MD_VTAG, Vtag, PbContent) when is_list(Vtag) -> 104 | PbContent#rpbcontent{vtag = Vtag}; 105 | encode_content_meta(?MD_LINKS, Links, PbContent) when is_list(Links) -> 106 | PbContent#rpbcontent{links = [encode_link(E) || E <- Links]}; 107 | encode_content_meta(?MD_LASTMOD, {MS,S,US}, PbContent) -> 108 | PbContent#rpbcontent{last_mod = 1000000*MS+S, last_mod_usecs = US}; 109 | encode_content_meta(?MD_USERMETA, UserMeta, PbContent) when is_list(UserMeta) -> 110 | PbContent#rpbcontent{usermeta = [encode_pair(E) || E <- UserMeta]}; 111 | encode_content_meta(?MD_INDEX, Indexes, PbContent) when is_list(Indexes) -> 112 | PbContent#rpbcontent{indexes = [encode_index_pair(E) || E <- Indexes]}; 113 | encode_content_meta(?MD_DELETED, DeletedVal, PbContent) -> 114 | PbContent#rpbcontent{deleted=header_val_to_bool(DeletedVal)}; 115 | encode_content_meta(_Key, _Value, PbContent) -> 116 | %% Ignore unknown metadata - need to add to RpbContent if it needs to make it 117 | %% to/from the client 118 | PbContent. 119 | 120 | %% @doc Return a boolean based on a header value. 121 | %% Representations of `true' return `true'; anything 122 | %% else returns `false'. 123 | -spec header_val_to_bool(term()) -> boolean(). 124 | header_val_to_bool(<<"true">>) -> 125 | true; 126 | header_val_to_bool("true") -> 127 | true; 128 | header_val_to_bool(true) -> 129 | true; 130 | header_val_to_bool(_) -> 131 | false. 132 | 133 | %% @doc Convert a list of rpbcontent pb messages to a list of [{MetaData,Value}] tuples 134 | -spec decode_contents(PBContents::[tuple()]) -> contents(). 135 | decode_contents(RpbContents) -> 136 | [decode_content(RpbContent) || RpbContent <- RpbContents]. 137 | 138 | -spec decode_content_meta(atom(), any(), #rpbcontent{}) -> [ {binary(), any()} ]. 139 | decode_content_meta(_, undefined, _Pb) -> 140 | []; 141 | decode_content_meta(_, [], _Pb) -> 142 | %% Repeated metadata fields that are empty lists need not be added 143 | %% to the decoded metadata. This previously resulted in 144 | %% type-conversion errors when using the JSON form of a 145 | %% riak_object. All of the other metadata types are primitive 146 | %% types. 147 | []; 148 | decode_content_meta(content_type, CType, _Pb) -> 149 | [{?MD_CTYPE, binary_to_list(CType)}]; 150 | decode_content_meta(charset, Charset, _Pb) -> 151 | [{?MD_CHARSET, binary_to_list(Charset)}]; 152 | decode_content_meta(encoding, Encoding, _Pb) -> 153 | [{?MD_ENCODING, binary_to_list(Encoding)}]; 154 | decode_content_meta(vtag, VTag, _Pb) -> 155 | [{?MD_VTAG, binary_to_list(VTag)}]; 156 | decode_content_meta(last_mod, LastMod, Pb) -> 157 | case Pb#rpbcontent.last_mod_usecs of 158 | undefined -> 159 | Usec = 0; 160 | Usec -> 161 | Usec 162 | end, 163 | Msec = LastMod div 1000000, 164 | Sec = LastMod rem 1000000, 165 | [{?MD_LASTMOD, {Msec,Sec,Usec}}]; 166 | decode_content_meta(links, Links1, _Pb) -> 167 | Links = [ decode_link(L) || L <- Links1 ], 168 | [{?MD_LINKS, Links}]; 169 | decode_content_meta(usermeta, PbUserMeta, _Pb) -> 170 | UserMeta = [decode_pair(E) || E <- PbUserMeta], 171 | [{?MD_USERMETA, UserMeta}]; 172 | decode_content_meta(indexes, PbIndexes, _Pb) -> 173 | Indexes = [decode_pair(E) || E <- PbIndexes], 174 | [{?MD_INDEX, Indexes}]; 175 | decode_content_meta(deleted, DeletedVal, _Pb) -> 176 | [{?MD_DELETED, DeletedVal}]. 177 | 178 | 179 | %% @doc Convert an rpccontent pb message to an erlang {MetaData,Value} tuple 180 | -spec decode_content(PBContent::tuple()) -> {metadata(), binary()}. 181 | decode_content(PbC) -> 182 | MD = decode_content_meta(content_type, PbC#rpbcontent.content_type, PbC) ++ 183 | decode_content_meta(charset, PbC#rpbcontent.charset, PbC) ++ 184 | decode_content_meta(encoding, PbC#rpbcontent.content_encoding, PbC) ++ 185 | decode_content_meta(vtag, PbC#rpbcontent.vtag, PbC) ++ 186 | decode_content_meta(links, PbC#rpbcontent.links, PbC) ++ 187 | decode_content_meta(last_mod, PbC#rpbcontent.last_mod, PbC) ++ 188 | decode_content_meta(usermeta, PbC#rpbcontent.usermeta, PbC) ++ 189 | decode_content_meta(indexes, PbC#rpbcontent.indexes, PbC) ++ 190 | decode_content_meta(deleted, PbC#rpbcontent.deleted, PbC), 191 | 192 | {dict:from_list(MD), PbC#rpbcontent.value}. 193 | 194 | %% @doc Convert {K,V} index entries into protocol buffers 195 | -spec encode_index_pair({binary(), integer() | binary()}) -> #rpbpair{}. 196 | encode_index_pair({K,V}) when is_integer(V) -> 197 | encode_pair({K, integer_to_list(V)}); 198 | encode_index_pair(E) -> 199 | encode_pair(E). 200 | 201 | %% @doc Convert {K,V} tuple to protocol buffers 202 | %% @equiv riak_pb_codec:encode_pair/1 203 | -spec encode_pair({Key::binary(), Value::any()}) -> #rpbpair{}. 204 | encode_pair(Pair) -> 205 | riak_pb_codec:encode_pair(Pair). 206 | 207 | %% @doc Convert RpbPair PB message to erlang {K,V} tuple 208 | %% @equiv riak_pb_codec:decode_pair/1 209 | -spec decode_pair(#rpbpair{}) -> {binary(), binary()}. 210 | decode_pair(PB) -> 211 | riak_pb_codec:decode_pair(PB). 212 | 213 | %% @doc Convert erlang link tuple to RpbLink PB message 214 | -spec encode_link({{binary(), binary()}, binary() | string()}) -> #rpblink{}. 215 | encode_link({{B,K},T}) -> 216 | #rpblink{bucket = B, key = K, tag = T}. 217 | 218 | %% @doc Convert RpbLink PB message to erlang link tuple 219 | -spec decode_link(PBLink::#rpblink{}) -> {{binary(), binary()}, binary()}. 220 | decode_link(#rpblink{bucket = B, key = K, tag = T}) -> 221 | {{B,K},T}. 222 | 223 | %% @doc Encode a symbolic or numeric quorum value into a Protocol 224 | %% Buffers value 225 | -spec encode_quorum(quorum()) -> non_neg_integer(). 226 | encode_quorum(Bin) when is_binary(Bin) -> encode_quorum(binary_to_existing_atom(Bin, latin1)); 227 | encode_quorum(one) -> ?RIAKPB_RW_ONE; 228 | encode_quorum(quorum) -> ?RIAKPB_RW_QUORUM; 229 | encode_quorum(all) -> ?RIAKPB_RW_ALL; 230 | encode_quorum(default) -> ?RIAKPB_RW_DEFAULT; 231 | encode_quorum(undefined) -> undefined; 232 | encode_quorum(I) when is_integer(I), I >= 0 -> I. 233 | 234 | %% @doc Decodes a Protocol Buffers value into a symbolic or numeric 235 | %% quorum. 236 | -spec decode_quorum(non_neg_integer()) -> quorum(). 237 | decode_quorum(?RIAKPB_RW_ONE) -> one; 238 | decode_quorum(?RIAKPB_RW_QUORUM) -> quorum; 239 | decode_quorum(?RIAKPB_RW_ALL) -> all; 240 | decode_quorum(?RIAKPB_RW_DEFAULT) -> default; 241 | decode_quorum(undefined) -> undefined; 242 | decode_quorum(I) when is_integer(I), I >= 0 -> I. 243 | 244 | %% @doc Convert preflist to RpbBucketKeyPreflist. 245 | -spec encode_apl_ann(preflist_with_pnum_ann()) -> 246 | PBPreflist::[#rpbbucketkeypreflistitem{}]. 247 | encode_apl_ann(Preflist) -> 248 | [encode_apl_item({PartitionNumber, Node}, T) || 249 | {{PartitionNumber, Node}, T} <- Preflist]. 250 | 251 | -spec encode_apl_item({non_neg_integer(), node()}, primary|fallback) -> 252 | #rpbbucketkeypreflistitem{}. 253 | encode_apl_item({PartitionNumber, Node}, primary) -> 254 | #rpbbucketkeypreflistitem{partition=PartitionNumber, 255 | node=riak_pb_codec:to_binary(Node), 256 | primary=riak_pb_codec:encode_bool(true)}; 257 | encode_apl_item({PartitionNumber, Node}, fallback) -> 258 | #rpbbucketkeypreflistitem{partition=PartitionNumber, 259 | node=riak_pb_codec:to_binary(Node), 260 | primary=riak_pb_codec:encode_bool(false)}. 261 | 262 | 263 | -ifdef(TEST). 264 | 265 | encode_apl_ann_test() -> 266 | Encoded = encode_apl_ann([{{1, 267 | 'dev5@127.0.0.1'}, 268 | primary}, 269 | {{2, 270 | 'dev6@127.0.0.1'}, 271 | primary}, 272 | {{3, 273 | 'dev3@127.0.0.1'}, 274 | fallback}]), 275 | ?assertEqual(Encoded, 276 | [{rpbbucketkeypreflistitem, 277 | 1,<<"dev5@127.0.0.1">>,true}, 278 | {rpbbucketkeypreflistitem, 279 | 2,<<"dev6@127.0.0.1">>,true}, 280 | {rpbbucketkeypreflistitem, 281 | 3,<<"dev3@127.0.0.1">>,false}]). 282 | 283 | -endif. 284 | -------------------------------------------------------------------------------- /src/riak_pb_messages.csv: -------------------------------------------------------------------------------- 1 | 0,RpbErrorResp,riak 2 | 1,RpbPingReq,riak 3 | 2,RpbPingResp,riak 4 | 3,RpbGetClientIdReq,riak_kv 5 | 4,RpbGetClientIdResp,riak_kv 6 | 5,RpbSetClientIdReq,riak_kv 7 | 6,RpbSetClientIdResp,riak_kv 8 | 7,RpbGetServerInfoReq,riak 9 | 8,RpbGetServerInfoResp,riak 10 | 9,RpbGetReq,riak_kv 11 | 10,RpbGetResp,riak_kv 12 | 11,RpbPutReq,riak_kv 13 | 12,RpbPutResp,riak_kv 14 | 13,RpbDelReq,riak_kv 15 | 14,RpbDelResp,riak_kv 16 | 15,RpbListBucketsReq,riak_kv 17 | 16,RpbListBucketsResp,riak_kv 18 | 17,RpbListKeysReq,riak_kv 19 | 18,RpbListKeysResp,riak_kv 20 | 19,RpbGetBucketReq,riak 21 | 20,RpbGetBucketResp,riak 22 | 21,RpbSetBucketReq,riak 23 | 22,RpbSetBucketResp,riak 24 | 23,RpbMapRedReq,riak_kv 25 | 24,RpbMapRedResp,riak_kv 26 | 25,RpbIndexReq,riak_kv 27 | 26,RpbIndexResp,riak_kv 28 | 27,RpbSearchQueryReq,riak_search 29 | 28,RpbSearchQueryResp,riak_search 30 | 29,RpbResetBucketReq,riak 31 | 30,RpbResetBucketResp,riak 32 | 31,RpbGetBucketTypeReq,riak 33 | 32,RpbSetBucketTypeReq,riak 34 | 33,RpbGetBucketKeyPreflistReq,riak_kv 35 | 34,RpbGetBucketKeyPreflistResp,riak_kv 36 | 40,RpbCSBucketReq,riak_kv 37 | 41,RpbCSBucketResp,riak_kv 38 | 42,RpbIndexBodyResp,riak_kv 39 | 50,RpbCounterUpdateReq,riak_kv 40 | 51,RpbCounterUpdateResp,riak_kv 41 | 52,RpbCounterGetReq,riak_kv 42 | 53,RpbCounterGetResp,riak_kv 43 | 54,RpbYokozunaIndexGetReq,riak_yokozuna 44 | 55,RpbYokozunaIndexGetResp,riak_yokozuna 45 | 56,RpbYokozunaIndexPutReq,riak_yokozuna 46 | 57,RpbYokozunaIndexDeleteReq,riak_yokozuna 47 | 58,RpbYokozunaSchemaGetReq,riak_yokozuna 48 | 59,RpbYokozunaSchemaGetResp,riak_yokozuna 49 | 60,RpbYokozunaSchemaPutReq,riak_yokozuna 50 | 70,RpbCoverageReq,riak_kv 51 | 71,RpbCoverageResp,riak_kv 52 | 80,DtFetchReq,riak_dt 53 | 81,DtFetchResp,riak_dt 54 | 82,DtUpdateReq,riak_dt 55 | 83,DtUpdateResp,riak_dt 56 | 90,TsQueryReq,riak_ts 57 | 91,TsQueryResp,riak_ts 58 | 92,TsPutReq,riak_ts 59 | 93,TsPutResp,riak_ts 60 | 94,TsDelReq,riak_ts 61 | 95,TsDelResp,riak_ts 62 | 96,TsGetReq,riak_ts 63 | 97,TsGetResp,riak_ts 64 | 98,TsListKeysReq,riak_ts 65 | 99,TsListKeysResp,riak_ts 66 | 100,TsCoverageReq,riak_ts 67 | 101,TsCoverageResp,riak_ts 68 | 102,TsCoverageEntry,riak_ts 69 | 103,TsRange,riak_ts 70 | 104,TsTtbMsg,riak_ts 71 | 128,RpbReplGetReq,riak_repl 72 | 129,RpbReplGetClusterIdReq,riak_repl 73 | 130,RpbReplGetClusterIdResp,riak_repl 74 | 200,RpbRTEReq,riak_kv 75 | 201,RpbRTEResp,riak_kv 76 | 202,RpbFetchReq,riak_kv 77 | 203,RpbFetchResp,riak_kv 78 | 204,RpbPushReq,riak_kv 79 | 205,RpbPushResp,riak_kv 80 | 206,RpbMembershipReq,riak_kv 81 | 207,RpbMembershipResp,riak_kv 82 | 210,RpbAaeFoldMergeRootNValReq,riak_kv 83 | 211,RpbAaeFoldMergeBranchNValReq,riak_kv 84 | 212,RpbAaeFoldFetchClocksNValReq,riak_kv 85 | 213,RpbAaeFoldMergeTreesRangeReq,riak_kv 86 | 214,RpbAaeFoldFetchClocksRangeReq,riak_kv 87 | 215,RpbAaeFoldFindKeysReq,riak_kv 88 | 216,RpbAaeFoldObjectStatsReq,riak_kv 89 | 217,RpbAaeFoldReplKeysReq,riak_kv 90 | 218,RpbAaeFoldFindTombsReq,riak_kv 91 | 219,RpbAaeFoldRepairKeysReq,riak_kv 92 | 220,RpbAaeFoldTreeResp,riak_kv 93 | 221,RpbAaeFoldKeyValueResp,riak_kv 94 | 222,RpbAaeFoldKeyCountResp,riak_kv 95 | 223,RpbAaeFoldListBucketsReq,riak_kv 96 | 224,RpbAaeFoldListBucketsResp,riak_kv 97 | 230,RpbAaeFoldReapTombsReq,riak_kv 98 | 231,RpbAaeFoldEraseKeysReq,riak_kv 99 | 253,RpbAuthReq,riak 100 | 254,RpbAuthResp,riak 101 | 255,RpbStartTls,riak 102 | -------------------------------------------------------------------------------- /src/riak_pb_search_codec.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riak_pb_search_codec: Protocol Buffers encoding/decoding helpers for 4 | %% Riak Search 5 | %% 6 | %% Copyright (c) 2012 Basho Technologies, Inc. All Rights Reserved. 7 | %% 8 | %% This file is provided to you under the Apache License, 9 | %% Version 2.0 (the "License"); you may not use this file 10 | %% except in compliance with the License. You may obtain 11 | %% a copy of the License at 12 | %% 13 | %% http://www.apache.org/licenses/LICENSE-2.0 14 | %% 15 | %% Unless required by applicable law or agreed to in writing, 16 | %% software distributed under the License is distributed on an 17 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 18 | %% KIND, either express or implied. See the License for the 19 | %% specific language governing permissions and limitations 20 | %% under the License. 21 | %% 22 | %% ------------------------------------------------------------------- 23 | 24 | %% @doc Utility functions for Protocol Buffers encoding and decoding 25 | %% of Riak Search-related messages. These are used inside the client 26 | %% and server code and do not normally need to be used in application 27 | %% code. 28 | -module(riak_pb_search_codec). 29 | 30 | -include("riak_search_pb.hrl"). 31 | 32 | -export([encode_search_doc/1, 33 | decode_search_doc/1]). 34 | 35 | -import(riak_pb_codec, [encode_pair/1, decode_pair/1]). 36 | 37 | %% @doc Encodes a property-list of indexed-document fields into a 38 | %% search-doc Protocol Buffers message. 39 | -spec encode_search_doc([{binary(), binary()}]) -> #rpbsearchdoc{}. 40 | encode_search_doc(PList) -> 41 | #rpbsearchdoc{fields=[ encode_pair(Pair) || Pair <- PList]}. 42 | 43 | %% @doc Decodes a Protocol Buffers message search-doc into proplist of 44 | %% document fields and values. 45 | -spec decode_search_doc(#rpbsearchdoc{}) -> [{binary(), binary()}]. 46 | decode_search_doc(#rpbsearchdoc{fields=Fields}) -> 47 | [ decode_pair(Pair) || Pair <- Fields ]. 48 | -------------------------------------------------------------------------------- /src/riak_pb_ts_codec.erl: -------------------------------------------------------------------------------- 1 | %% 2 | %% riak_pb_ts_codec.erl: protocol buffer utility functions for Riak TS messages 3 | %% 4 | %% Copyright (c) 2015 Basho Technologies, Inc. All Rights Reserved. 5 | %% 6 | %% This file is provided to you under the Apache License, 7 | %% Version 2.0 (the "License"); you may not use this file 8 | %% except in compliance with the License. You may obtain 9 | %% a copy of the License at 10 | %% 11 | %% http://www.apache.org/licenses/LICENSE-2.0 12 | %% 13 | %% Unless required by applicable law or agreed to in writing, 14 | %% software distributed under the License is distributed on an 15 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | %% KIND, either express or implied. See the License for the 17 | %% specific language governing permissions and limitations 18 | %% under the License. 19 | %% 20 | %% ------------------------------------------------------------------- 21 | 22 | %% @doc Utility functions for decoding and encoding Protocol Buffers 23 | %% messages related to Riak TS. 24 | 25 | -module(riak_pb_ts_codec). 26 | 27 | -include("riak_ts_pb.hrl"). 28 | 29 | -export([encode_columnnames/1, 30 | encode_rows/2, 31 | encode_rows_non_strict/1, 32 | encode_columns/2, 33 | decode_rows/1, 34 | encode_cells/1, 35 | encode_cells_non_strict/1, 36 | decode_cells/1, 37 | encode_field_type/1, 38 | encode_cover_list/1, 39 | decode_cover_list/1]). 40 | 41 | 42 | -type tsrow() :: #tsrow{}. 43 | -export_type([tsrow/0]). 44 | 45 | %% types existing between us and eleveldb 46 | -type ldbvalue() :: binary() | number() | boolean() | list(). 47 | 48 | %% types of #tscell.xxx_value fields, constrained by what protobuf messages accept 49 | %% -type pbvalue() :: binary() | integer() | boolean(). 50 | -export_type([ldbvalue/0]). 51 | 52 | %% Column names are binary only. 53 | -type tscolumnname() :: binary(). 54 | %% Possible column type values supported and returned from the timeseries DDL. 55 | -type tscolumntype() :: varchar | sint64 | timestamp | boolean | double. 56 | %% Possible column type values that protocol buffers supports for enumeration purposes. 57 | -type tscolumntypePB() :: 'VARCHAR' | 'SINT64' | 'TIMESTAMP' | 'BOOLEAN' | 'DOUBLE'. 58 | -export_type([tscolumnname/0, tscolumntype/0, tscolumntypePB/0]). 59 | 60 | %% @doc Convert a list of column names to partial #tscolumndescription records. 61 | -spec encode_columnnames([tscolumnname()]) -> [#tscolumndescription{}]. 62 | encode_columnnames(ColumnNames) -> 63 | [#tscolumndescription{name = C} || C <- ColumnNames]. 64 | 65 | %% @doc Convert time series field type atoms returned from the DDL modules 66 | %% into Protobuf compatible upper case atoms. 67 | -spec encode_field_type(tscolumntype()) -> atom(). 68 | encode_field_type(varchar) -> 69 | 'VARCHAR'; 70 | encode_field_type(sint64) -> 71 | 'SINT64'; 72 | encode_field_type(double) -> 73 | 'DOUBLE'; 74 | encode_field_type(timestamp) -> 75 | 'TIMESTAMP'; 76 | encode_field_type(boolean) -> 77 | 'BOOLEAN'. 78 | 79 | %% @doc Encode a set of time series rows from an internal format to the #tsrow record format. 80 | %% Takes a list of column types, and a list of rows. 81 | %% Each row is represented as a list of ldbvalue(). 82 | %% An error is returned if any of the `Rows` individual row length do not match the length of the `ColumnTypes` list. 83 | %% @end 84 | -spec encode_rows([tscolumntype()], [{ldbvalue()}] | [[ldbvalue()]]) -> [#tsrow{}]. 85 | encode_rows(ColumnTypes, Rows) -> 86 | [encode_row(ColumnTypes, Row) || Row <- Rows]. 87 | 88 | %% @doc Only for encoding rows for PUTs on the erlang client. 89 | %% Will not properly encode timestamp #tscell{} records, 90 | %% but this is OK for this use case since the values 91 | %% get cast in the order of: 92 | %% lvldbvalue() -> #tscell{} -> lvldbvalue(). 93 | %% THEREFORE no info is lost for these cases. 94 | %% @end 95 | encode_rows_non_strict(Rows) -> 96 | [encode_row_non_strict(Row) || Row <- Rows]. 97 | 98 | -spec encode_columns([binary()], [riak_pb_ts_codec:tscolumntype()]) -> 99 | [#tscolumndescription{}]. 100 | encode_columns(ColumnNames, ColumnTypes) -> 101 | [#tscolumndescription{name = Name, type = encode_field_type(Type)} 102 | || {Name, Type} <- lists:zip(ColumnNames, ColumnTypes)]. 103 | 104 | 105 | %% @doc Decode a list of timeseries #tsrow{} to a list of tuples. 106 | %% Each row is converted through `decode_cells/1`, and the list 107 | %% of ldbvalue() is converted to a tuple of ldbvalue(). 108 | %% @end 109 | -spec decode_rows([#tsrow{}]) -> [{ldbvalue()}]. 110 | decode_rows(Rows) -> 111 | [list_to_tuple(decode_cells(Cells)) || #tsrow{cells = Cells} <- Rows]. 112 | 113 | -spec encode_cells([{tscolumntype(), ldbvalue()}]) -> [#tscell{}]. 114 | encode_cells(Cells) -> 115 | [encode_cell(C) || C <- Cells]. 116 | 117 | %% @doc Decode a list of timeseries #tscell{} to a list of ldbvalue(). 118 | -spec decode_cells([#tscell{}]) -> [ldbvalue()]. 119 | decode_cells(Cells) -> 120 | decode_cells(Cells, []). 121 | 122 | %% --------------------------------------- 123 | %% local functions 124 | %% --------------------------------------- 125 | 126 | -spec encode_row([tscolumntype()], [ldbvalue()] | {ldbvalue()}) -> #tsrow{}. 127 | encode_row(ColumnTypes, RowCells) when is_tuple(RowCells) -> 128 | encode_row(ColumnTypes, tuple_to_list(RowCells)); 129 | encode_row(ColumnTypes, RowCells) when is_list(RowCells), length(ColumnTypes) =:= length(RowCells) -> 130 | #tsrow{cells = [encode_cell(ColumnTypeCell) || 131 | ColumnTypeCell <- lists:zip(ColumnTypes, RowCells)]}. 132 | 133 | %% @doc Only for encoding rows for PUTs on the erlang client. 134 | %% Will not properly encode timestamp #tscell{} records, 135 | %% but this is OK for these use cases since the values 136 | %% get cast in the order of: 137 | %% lvldbvalue() -> #tscell{} -> lvldbvalue(). 138 | %% THEREFORE no info is lost for these cases. 139 | %% @end 140 | -spec encode_row_non_strict([ldbvalue()]) -> #tsrow{}. 141 | encode_row_non_strict(RowCells) -> 142 | #tsrow{cells = encode_cells_non_strict(RowCells)}. 143 | 144 | %% @doc Only for encoding cells for PUTs on the erlang client, 145 | %% and Key cells for get / delete requests. 146 | %% Will not properly encode timestamp #tscell{} records, 147 | %% but this is OK for these use cases since the values 148 | %% get cast in the order of: 149 | %% lvldbvalue() -> #tscell{} -> lvldbvalue(). 150 | %% THEREFORE no info is lost for these cases. 151 | %% @end 152 | -spec encode_cells_non_strict([ldbvalue()] | {ldbvalue()}) -> [#tscell{}]. 153 | encode_cells_non_strict(Cells) when is_tuple(Cells) -> 154 | encode_cells_non_strict(tuple_to_list(Cells)); 155 | encode_cells_non_strict(Cells) when is_list(Cells) -> 156 | [encode_cell_non_strict(Cell) || Cell <- Cells]. 157 | 158 | -spec encode_cell({tscolumntype(), ldbvalue()}) -> #tscell{}. 159 | encode_cell({varchar, V}) when is_binary(V) -> 160 | #tscell{varchar_value = V}; 161 | encode_cell({sint64, V}) when is_integer(V) -> 162 | #tscell{sint64_value = V}; 163 | encode_cell({double, V}) when is_float(V) -> 164 | #tscell{double_value = V}; 165 | encode_cell({timestamp, V}) when is_integer(V) -> 166 | #tscell{timestamp_value = V}; 167 | encode_cell({boolean, V}) when is_boolean(V) -> 168 | #tscell{boolean_value = V}; 169 | encode_cell({_ColumnType, undefined}) -> 170 | #tscell{}; 171 | %% NULL Cell 172 | %% TODO: represent null cells by something other than an empty list. emptyTsCell atom maybe? 173 | encode_cell({_ColumnType, []}) -> 174 | #tscell{}. 175 | 176 | %% @doc Only for encoding rows for PUTs on the erlang client, 177 | %% and Key cells for get / delete requests. 178 | %% Will not properly encode timestamp #tscell{} records, 179 | %% but this is OK for these use cases since the values 180 | %% get cast in the order of: 181 | %% lvldbvalue() -> #tscell{} -> lvldbvalue(). 182 | %% THEREFORE no info is lost for these cases. 183 | %% @end 184 | -spec encode_cell_non_strict(ldbvalue()) -> #tscell{}. 185 | encode_cell_non_strict(V) when is_binary(V) -> 186 | #tscell{varchar_value = V}; 187 | encode_cell_non_strict(V) when is_integer(V) -> 188 | #tscell{sint64_value = V}; 189 | encode_cell_non_strict(V) when is_float(V) -> 190 | #tscell{double_value = V}; 191 | encode_cell_non_strict(V) when is_boolean(V) -> 192 | #tscell{boolean_value = V}; 193 | encode_cell_non_strict(undefined) -> 194 | #tscell{}; 195 | %% NULL Cell 196 | %% TODO: represent null cells by something other than an empty list. emptyTsCell atom maybe? 197 | encode_cell_non_strict([]) -> 198 | #tscell{}. 199 | 200 | -spec decode_cells([#tscell{}], [ldbvalue()]) -> [ldbvalue()]. 201 | decode_cells([], Acc) -> 202 | lists:reverse(Acc); 203 | decode_cells([#tscell{varchar_value = Bin, 204 | sint64_value = undefined, 205 | timestamp_value = undefined, 206 | boolean_value = undefined, 207 | double_value = undefined} | T], Acc) 208 | when is_binary(Bin) -> 209 | decode_cells(T, [Bin | Acc]); 210 | decode_cells([#tscell{varchar_value = undefined, 211 | sint64_value = Int, 212 | timestamp_value = undefined, 213 | boolean_value = undefined, 214 | double_value = undefined} | T], Acc) 215 | when is_integer(Int) -> 216 | decode_cells(T, [Int | Acc]); 217 | decode_cells([#tscell{varchar_value = undefined, 218 | sint64_value = undefined, 219 | timestamp_value = Timestamp, 220 | boolean_value = undefined, 221 | double_value = undefined} | T], Acc) 222 | when is_integer(Timestamp) -> 223 | decode_cells(T, [Timestamp | Acc]); 224 | decode_cells([#tscell{varchar_value = undefined, 225 | sint64_value = undefined, 226 | timestamp_value = undefined, 227 | boolean_value = Bool, 228 | double_value = undefined} | T], Acc) 229 | when is_boolean(Bool) -> 230 | decode_cells(T, [Bool | Acc]); 231 | decode_cells([#tscell{varchar_value = undefined, 232 | sint64_value = undefined, 233 | timestamp_value = undefined, 234 | boolean_value = undefined, 235 | double_value = Double} | T], Acc) 236 | when is_float(Double) -> 237 | decode_cells(T, [Double | Acc]); 238 | decode_cells([#tscell{varchar_value = undefined, 239 | sint64_value = undefined, 240 | timestamp_value = undefined, 241 | boolean_value = undefined, 242 | double_value = undefined} | T], Acc) -> 243 | %% NULL Cell. 244 | %% TODO: represent null cells by something other than an empty list. emptyTsCell atom maybe? 245 | decode_cells(T, [[] | Acc]). 246 | 247 | 248 | 249 | %% Copied and modified from riak_kv_pb_coverage:convert_list. Would 250 | %% be nice to collapse them back together, probably with a closure, 251 | %% but time and effort. 252 | -type ts_range() :: {FieldName::binary(), 253 | {{StartVal::integer(), StartIncl::boolean()}, 254 | {EndVal::integer(), EndIncl::boolean()}}}. 255 | 256 | -spec encode_cover_list([{{IP::string(), Port::non_neg_integer()}, 257 | Context::binary(), 258 | ts_range(), 259 | SQLText::binary()}]) -> [#tscoverageentry{}]. 260 | encode_cover_list(Entries) -> 261 | [#tscoverageentry{ip = IP, port = Port, 262 | cover_context = Context, 263 | range = encode_ts_range({Range, SQLText})} 264 | || {{IP, Port}, Context, Range, SQLText} <- Entries]. 265 | 266 | -spec decode_cover_list([#tscoverageentry{}]) -> 267 | [{{IP::string(), Port::non_neg_integer()}, 268 | CoverContext::binary(), ts_range(), Text::binary()}]. 269 | decode_cover_list(Entries) -> 270 | [begin 271 | {RangeStruct, Text} = decode_ts_range(Range), 272 | {{IP, Port}, CoverContext, RangeStruct, Text} 273 | end || #tscoverageentry{ip = IP, port = Port, 274 | cover_context = CoverContext, 275 | range = Range} <- Entries]. 276 | 277 | -spec encode_ts_range({ts_range(), binary()}) -> #tsrange{}. 278 | encode_ts_range({{FieldName, {{StartVal, StartIncl}, {EndVal, EndIncl}}}, Text}) -> 279 | #tsrange{field_name = FieldName, 280 | lower_bound = StartVal, 281 | lower_bound_inclusive = StartIncl, 282 | upper_bound = EndVal, 283 | upper_bound_inclusive = EndIncl, 284 | desc = Text 285 | }. 286 | 287 | -spec decode_ts_range(#tsrange{}) -> {ts_range(), binary()}. 288 | decode_ts_range(#tsrange{field_name = FieldName, 289 | lower_bound = StartVal, 290 | lower_bound_inclusive = StartIncl, 291 | upper_bound = EndVal, 292 | upper_bound_inclusive = EndIncl, 293 | desc = Text}) -> 294 | {{FieldName, {{StartVal, StartIncl}, {EndVal, EndIncl}}}, Text}. 295 | 296 | 297 | 298 | -ifdef(TEST). 299 | -include_lib("eunit/include/eunit.hrl"). 300 | 301 | encode_cells_test() -> 302 | %% Correct cells 303 | ?assertEqual(#tscell{varchar_value = <<"Foo">>}, encode_cell({varchar, <<"Foo">>})), 304 | ?assertEqual(#tscell{sint64_value = 64}, encode_cell({sint64, 64})), 305 | ?assertEqual(#tscell{timestamp_value = 64}, encode_cell({timestamp, 64})), 306 | ?assertEqual(#tscell{boolean_value = true}, encode_cell({boolean, true})), 307 | ?assertEqual(#tscell{double_value = 42.0}, encode_cell({double, 42.0})), 308 | ?assertEqual(#tscell{boolean_value = false}, encode_cell({boolean, false})), 309 | 310 | %% Null Cells 311 | ?assertEqual(#tscell{}, encode_cell({varchar, []})), 312 | ?assertEqual(#tscell{}, encode_cell({sint64, []})), 313 | ?assertEqual(#tscell{}, encode_cell({timestamp, []})), 314 | ?assertEqual(#tscell{}, encode_cell({boolean, []})), 315 | ?assertEqual(#tscell{}, encode_cell({double, []})), 316 | 317 | %% Just plain wrong Cells 318 | ?assertError(function_clause, encode_cell({varchar, 42})), 319 | ?assertError(function_clause, encode_cell({varchar, true})), 320 | ?assertError(function_clause, encode_cell({sint64, <<"42">>})), 321 | ?assertError(function_clause, encode_cell({sint64, true})), 322 | ?assertError(function_clause, encode_cell({boolean, <<"42">>})), 323 | ?assertError(function_clause, encode_cell({boolean, 42})). 324 | 325 | encode_row_test() -> 326 | ?assertEqual( 327 | #tsrow{cells = [ 328 | #tscell{varchar_value = <<"Foo">>}, 329 | #tscell{sint64_value = 64}, 330 | #tscell{timestamp_value = 42}, 331 | #tscell{boolean_value = false}, 332 | #tscell{double_value = 42.2}, 333 | #tscell{} 334 | ]}, 335 | encode_row( 336 | [varchar, sint64, timestamp, boolean, double, varchar], 337 | [<<"Foo">>, 64, 42, false, 42.2, []] 338 | )), 339 | ?assertError(function_clause, encode_row([], [<<"Foo">>, 64, 42, false, 42.2, []])), 340 | ?assertError(function_clause, encode_row([varchar, sint64, timestamp], [<<"Foo">>, 64, 42, false, 42.2, []])). 341 | 342 | encode_rows_test() -> 343 | ?assertEqual( 344 | [ 345 | #tsrow{cells = [ 346 | #tscell{varchar_value = <<"Foo">>}, 347 | #tscell{sint64_value = 30} 348 | ]}, 349 | #tsrow{cells = [ 350 | #tscell{varchar_value = <<"Bar">>}, 351 | #tscell{sint64_value = 40} 352 | ]} 353 | ], 354 | encode_rows([varchar, sint64], [[<<"Foo">>, 30], [<<"Bar">>, 40]]) 355 | ), 356 | ?assertError(function_clause, encode_rows([], [[<<"Foo">>, 30], [<<"Bar">>, 40]])), 357 | ?assertError(function_clause, encode_rows([varchar], [[<<"Foo">>, 30], [<<"Bar">>, 40]])). 358 | 359 | encode_field_type_test() -> 360 | ?assertEqual('VARCHAR', encode_field_type(varchar)), 361 | ?assertEqual('SINT64',encode_field_type(sint64)), 362 | ?assertEqual('TIMESTAMP',encode_field_type(timestamp)), 363 | ?assertEqual('BOOLEAN',encode_field_type(boolean)), 364 | ?assertEqual('DOUBLE',encode_field_type(double)). 365 | 366 | decode_cell_test() -> 367 | ?assertEqual([<<"Foo">>], decode_cells([#tscell{varchar_value = <<"Foo">>}],[])), 368 | ?assertEqual([42], decode_cells([#tscell{sint64_value = 42}],[])), 369 | ?assertEqual([64], decode_cells([#tscell{timestamp_value = 64}],[])), 370 | ?assertEqual([false], decode_cells([#tscell{boolean_value = false}],[])), 371 | ?assertEqual([42.2], decode_cells([#tscell{double_value = 42.2}],[])), 372 | ?assertEqual([[]], decode_cells([#tscell{}],[])), 373 | ?assertEqual( 374 | [<<"Bar">>, 80, []], 375 | decode_cells([#tscell{varchar_value = <<"Bar">>}, #tscell{sint64_value = 80}, #tscell{}],[])), 376 | ?assertError( 377 | function_clause, 378 | decode_cells([#tscell{varchar_value = <<"Foo">>, sint64_value = 30}],[])). 379 | 380 | decode_cells_test() -> 381 | ?assertEqual( 382 | [<<"Bar">>, 80, []], 383 | decode_cells([#tscell{varchar_value = <<"Bar">>}, #tscell{sint64_value = 80}, #tscell{}])), 384 | ?assertError( 385 | function_clause, 386 | decode_cells([#tscell{varchar_value = <<"Foo">>, sint64_value = 30}])). 387 | 388 | decode_rows_test() -> 389 | ?assertEqual( 390 | [{<<"Bar">>, 80, []}, {<<"Baz">>, 90, false}], 391 | decode_rows([ 392 | #tsrow{cells = [#tscell{varchar_value = <<"Bar">>}, #tscell{sint64_value = 80}, #tscell{}]}, 393 | #tsrow{cells = [#tscell{varchar_value = <<"Baz">>}, #tscell{sint64_value = 90}, #tscell{boolean_value = false}]}])). 394 | 395 | -endif. 396 | -------------------------------------------------------------------------------- /src/riak_search.proto: -------------------------------------------------------------------------------- 1 | /* ------------------------------------------------------------------- 2 | ** 3 | ** riak_search.proto: Protocol buffers for Riak Search 4 | ** 5 | ** Copyright (c) 2012 Basho Technologies, Inc. All Rights Reserved. 6 | ** 7 | ** This file is provided to you under the Apache License, 8 | ** Version 2.0 (the "License"); you may not use this file 9 | ** except in compliance with the License. You may obtain 10 | ** a copy of the License at 11 | ** 12 | ** http://www.apache.org/licenses/LICENSE-2.0 13 | ** 14 | ** Unless required by applicable law or agreed to in writing, 15 | ** software distributed under the License is distributed on an 16 | ** "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | ** KIND, either express or implied. See the License for the 18 | ** specific language governing permissions and limitations 19 | ** under the License. 20 | ** 21 | ** ------------------------------------------------------------------- 22 | */ 23 | 24 | /* 25 | ** Revision: 1.4 26 | */ 27 | 28 | import "riak.proto"; 29 | 30 | // java package specifiers 31 | option java_package = "com.basho.riak.protobuf"; 32 | option java_outer_classname = "RiakSearchPB"; 33 | 34 | message RpbSearchDoc { 35 | repeated RpbPair fields = 1; 36 | } 37 | 38 | message RpbSearchQueryReq { 39 | required bytes q = 1; // Query string 40 | required bytes index = 2; // Index 41 | optional uint32 rows = 3; // Limit rows 42 | optional uint32 start = 4; // Starting offset 43 | optional bytes sort = 5; // Sort order 44 | optional bytes filter = 6; // Inline fields filtering query 45 | optional bytes df = 7; // Default field 46 | optional bytes op = 8; // Default op 47 | repeated bytes fl = 9; // Return fields limit (for ids only, generally) 48 | optional bytes presort = 10; // Presort (key / score) 49 | } 50 | 51 | message RpbSearchQueryResp { 52 | repeated RpbSearchDoc docs = 1; // Result documents 53 | optional float max_score = 2; // Maximum score 54 | optional uint32 num_found = 3; // Number of results 55 | } 56 | -------------------------------------------------------------------------------- /src/riak_ts.proto: -------------------------------------------------------------------------------- 1 | /* ------------------------------------------------------------------- 2 | ** 3 | ** riak_ts.proto: Protocol buffers for riak KV (timeseries-specific messages) 4 | ** 5 | ** Copyright (c) 2015 Basho Technologies, Inc. All Rights Reserved. 6 | ** 7 | ** This file is provided to you under the Apache License, 8 | ** Version 2.0 (the "License"); you may not use this file 9 | ** except in compliance with the License. You may obtain 10 | ** a copy of the License at 11 | ** 12 | ** http://www.apache.org/licenses/LICENSE-2.0 13 | ** 14 | ** Unless required by applicable law or agreed to in writing, 15 | ** software distributed under the License is distributed on an 16 | ** "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | ** KIND, either express or implied. See the License for the 18 | ** specific language governing permissions and limitations 19 | ** under the License. 20 | ** 21 | ** ------------------------------------------------------------------- 22 | */ 23 | 24 | /* 25 | ** Revision: 1.0rc7 26 | */ 27 | 28 | // Java package specifiers 29 | option java_package = "com.basho.riak.protobuf"; 30 | option java_outer_classname = "RiakTsPB"; 31 | 32 | import "riak.proto"; // for RpbPair 33 | 34 | 35 | // Dispatch a query to Riak 36 | message TsQueryReq { 37 | // left optional to support parameterized queries in the future 38 | optional TsInterpolation query = 1; 39 | optional bool stream = 2 [default = false]; 40 | optional bytes cover_context = 3; // chopped up coverage plan per-req 41 | } 42 | 43 | message TsQueryResp { 44 | repeated TsColumnDescription columns = 1; 45 | repeated TsRow rows = 2; // 0 to n rows 46 | optional bool done = 3 [default = true]; 47 | } 48 | 49 | message TsGetReq { 50 | required bytes table = 1; 51 | repeated TsCell key = 2; 52 | optional uint32 timeout = 3; 53 | } 54 | 55 | message TsGetResp { 56 | repeated TsColumnDescription columns = 1; 57 | repeated TsRow rows = 2; // 0 or 1 rows 58 | } 59 | 60 | 61 | message TsPutReq { 62 | required bytes table = 1; 63 | 64 | // optional: omitting it should use table order 65 | repeated TsColumnDescription columns = 2; 66 | 67 | repeated TsRow rows = 3; 68 | } 69 | 70 | message TsPutResp { 71 | 72 | } 73 | 74 | message TsDelReq { 75 | required bytes table = 1; 76 | repeated TsCell key = 2; 77 | optional bytes vclock = 3; 78 | optional uint32 timeout = 4; 79 | } 80 | 81 | message TsDelResp { 82 | 83 | } 84 | 85 | message TsInterpolation { 86 | required bytes base = 1; 87 | repeated RpbPair interpolations = 2; 88 | } 89 | 90 | enum TsColumnType { 91 | VARCHAR = 0; 92 | SINT64 = 1; 93 | DOUBLE = 2; 94 | TIMESTAMP = 3; 95 | BOOLEAN = 4; 96 | } 97 | 98 | message TsColumnDescription { 99 | required bytes name = 1; 100 | optional TsColumnType type = 2; 101 | } 102 | 103 | message TsRow { 104 | repeated TsCell cells = 1; 105 | } 106 | 107 | message TsCell { 108 | optional bytes varchar_value = 1; 109 | optional sint64 sint64_value = 2; 110 | optional sint64 timestamp_value = 3; 111 | optional bool boolean_value = 4; 112 | optional double double_value = 5; 113 | } 114 | 115 | message TsListKeysReq { 116 | required bytes table = 1; 117 | optional uint32 timeout = 2; 118 | } 119 | 120 | message TsListKeysResp { 121 | repeated TsRow keys = 1; 122 | optional bool done = 2; 123 | } 124 | 125 | // Request a segmented coverage plan for this query 126 | message TsCoverageReq { 127 | // left optional to support parameterized queries in the future 128 | optional TsInterpolation query = 1; 129 | required bytes table = 2; 130 | optional bytes replace_cover = 3; // For failure recovery 131 | repeated bytes unavailable_cover = 4; // Other coverage contexts that have failed to assist Riak in deciding what nodes to avoid 132 | } 133 | 134 | // Segmented TS coverage plan response 135 | message TsCoverageResp { 136 | repeated TsCoverageEntry entries = 1; 137 | } 138 | 139 | // Segment of a TS coverage plan 140 | message TsCoverageEntry { 141 | required bytes ip = 1; 142 | required uint32 port = 2; 143 | required bytes cover_context = 3; // Opaque context to pass into follow-up request 144 | optional TsRange range = 4; // Might be other types of coverage queries/responses 145 | } 146 | 147 | // Each prospective subquery has a range of valid time values 148 | message TsRange { 149 | required bytes field_name = 1; 150 | required sint64 lower_bound = 2; 151 | required bool lower_bound_inclusive = 3; 152 | required sint64 upper_bound = 4; 153 | required bool upper_bound_inclusive = 5; 154 | required bytes desc = 6; // Some human readable description of the time range 155 | } 156 | -------------------------------------------------------------------------------- /src/riak_ttb_codec.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riak_ttb_codec.erl: term-to-binary codec functions for 4 | %% Riak messages 5 | %% 6 | %% Copyright (c) 2015 Basho Technologies, Inc. All Rights Reserved. 7 | %% 8 | %% This file is provided to you under the Apache License, 9 | %% Version 2.0 (the "License"); you may not use this file 10 | %% except in compliance with the License. You may obtain 11 | %% a copy of the License at 12 | %% 13 | %% http://www.apache.org/licenses/LICENSE-2.0 14 | %% 15 | %% Unless required by applicable law or agreed to in writing, 16 | %% software distributed under the License is distributed on an 17 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 18 | %% KIND, either express or implied. See the License for the 19 | %% specific language governing permissions and limitations 20 | %% under the License. 21 | %% 22 | %% ------------------------------------------------------------------- 23 | 24 | %% @doc Codec for Riak term-to-binary messages. 25 | 26 | -module(riak_ttb_codec). 27 | 28 | -include("riak_ts_ttb.hrl"). 29 | 30 | -export([encode/1, 31 | decode/1]). 32 | 33 | %% ------------------------------------------------------------ 34 | %% Encode for TTB simply converts any strings to binary and encodes to 35 | %% erlang binary format 36 | %% ------------------------------------------------------------ 37 | 38 | encode(Msg) -> 39 | [?TTB_MSG_CODE, term_to_binary(Msg)]. 40 | 41 | 42 | %% ------------------------------------------------------------ 43 | %% Decode does the reverse 44 | %% ------------------------------------------------------------ 45 | 46 | decode(MsgData) -> 47 | return_resp(binary_to_term(MsgData)). 48 | 49 | %% ------------------------------------------------------------ 50 | %% But if the decoded response is empty, just return the atom 51 | %% identifying the message. This mimics the behavior of the PB 52 | %% decoder, which simply returns msg_type(msg_code) if the message 53 | %% body is empty 54 | %% ------------------------------------------------------------ 55 | 56 | return_resp({Atom, <<>>}) -> 57 | Atom; 58 | return_resp(Resp) -> 59 | Resp. 60 | -------------------------------------------------------------------------------- /src/riak_yokozuna.proto: -------------------------------------------------------------------------------- 1 | /* ------------------------------------------------------------------- 2 | ** 3 | ** riak_yokozuna.proto: Protocol buffers for Yokozuna 4 | ** 5 | ** Copyright (c) 2013 Basho Technologies, Inc. All Rights Reserved. 6 | ** 7 | ** This file is provided to you under the Apache License, 8 | ** Version 2.0 (the "License"); you may not use this file 9 | ** except in compliance with the License. You may obtain 10 | ** a copy of the License at 11 | ** 12 | ** http://www.apache.org/licenses/LICENSE-2.0 13 | ** 14 | ** Unless required by applicable law or agreed to in writing, 15 | ** software distributed under the License is distributed on an 16 | ** "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | ** KIND, either express or implied. See the License for the 18 | ** specific language governing permissions and limitations 19 | ** under the License. 20 | ** 21 | ** ------------------------------------------------------------------- 22 | */ 23 | 24 | /* 25 | ** Revision: 2.0 26 | */ 27 | 28 | // java package specifiers 29 | option java_package = "com.basho.riak.protobuf"; 30 | option java_outer_classname = "RiakYokozunaPB"; 31 | 32 | // Index queries 33 | 34 | message RpbYokozunaIndex { 35 | required bytes name = 1; // Index name 36 | optional bytes schema = 2; // Schema name 37 | optional uint32 n_val = 3; // N value 38 | } 39 | 40 | // GET request - If a name is given, return matching index, else return all 41 | message RpbYokozunaIndexGetReq { 42 | optional bytes name = 1; // Index name 43 | } 44 | 45 | message RpbYokozunaIndexGetResp { 46 | repeated RpbYokozunaIndex index = 1; 47 | } 48 | 49 | // PUT request - Create a new index 50 | message RpbYokozunaIndexPutReq { 51 | required RpbYokozunaIndex index = 1; 52 | optional uint32 timeout = 2; // Timeout value 53 | } 54 | 55 | // DELETE request - Remove an index 56 | message RpbYokozunaIndexDeleteReq { 57 | required bytes name = 1; // Index name 58 | } 59 | 60 | // Schema queries 61 | 62 | message RpbYokozunaSchema { 63 | required bytes name = 1; // Index name 64 | optional bytes content = 2; // Schema data 65 | } 66 | 67 | // PUT request - create or potentially update a new schema 68 | message RpbYokozunaSchemaPutReq { 69 | required RpbYokozunaSchema schema = 1; 70 | } 71 | 72 | // GET request - Return matching schema by name 73 | message RpbYokozunaSchemaGetReq { 74 | required bytes name = 1; // Schema name 75 | } 76 | 77 | message RpbYokozunaSchemaGetResp { 78 | required RpbYokozunaSchema schema = 1; 79 | } 80 | -------------------------------------------------------------------------------- /test/encoding_test.erl: -------------------------------------------------------------------------------- 1 | -module(encoding_test). 2 | -compile([export_all, nowarn_export_all]). 3 | -include_lib("eunit/include/eunit.hrl"). 4 | -include("riak_pb_kv_codec.hrl"). 5 | -include("riak_dt_pb.hrl"). 6 | 7 | pb_test_() -> 8 | [{"content encode decode", 9 | ?_test(begin 10 | MetaData = dict:from_list( 11 | [{?MD_CTYPE, "ctype"}, 12 | {?MD_CHARSET, "charset"}, 13 | {?MD_ENCODING, "encoding"}, 14 | {?MD_VTAG, "vtag"}, 15 | {?MD_LINKS, [{{<<"b1">>, <<"k1">>}, <<"v1">>}, 16 | {{<<"b2">>, <<"k2">>}, <<"v2">>} 17 | ]}, 18 | {?MD_LASTMOD, {1, 2, 3}}, 19 | {?MD_USERMETA, [{<<"X-Riak-Meta-MyMetaData1">>, <<"here it is">>}, 20 | {<<"X-Riak-Meta-MoreMd">>, <<"have some more">>}, 21 | {<<"X-Riak-Meta-EvenMoreMd">>, term_to_binary({a,b,c,d,e,f})} 22 | ]}, 23 | {?MD_INDEX, [{<<"index_bin">>, <<"foo">>}]}, 24 | {?MD_DELETED, true} 25 | ]), 26 | Value = <<"test value">>, 27 | {MetaData2, Value2} = riak_pb_kv_codec:decode_content( 28 | riak_kv_pb:decode_msg( 29 | iolist_to_binary(riak_kv_pb:encode_msg( 30 | riak_pb_kv_codec:encode_content({MetaData, Value}))), 31 | rpbcontent)), 32 | ?assertEqual(lists:sort(dict:to_list(MetaData)), lists:sort(dict:to_list(MetaData2))), 33 | ?assertEqual(Value, Value2) 34 | end)}, 35 | {"deleted header encode decode", 36 | ?_test(begin 37 | InputMD = [dict:from_list([{?MD_DELETED, DelVal}]) || 38 | DelVal <- [true, "true", false, <<"rubbish">>]], 39 | Value = <<"test value">>, 40 | {OutputMD, _} = lists:unzip( 41 | [riak_pb_kv_codec:decode_content( 42 | riak_kv_pb:decode_msg( 43 | iolist_to_binary(riak_kv_pb:encode_msg( 44 | riak_pb_kv_codec:encode_content({MD, Value}))), 45 | rpbcontent)) || 46 | MD <- InputMD]), 47 | MdSame1 = (lists:sort(dict:to_list(lists:nth(1, OutputMD))) =:= 48 | lists:sort(dict:to_list(lists:nth(2, OutputMD)))), 49 | MdSame2 = (lists:sort(dict:to_list(lists:nth(3, OutputMD))) =:= 50 | lists:sort(dict:to_list(lists:nth(4, OutputMD)))), 51 | ?assertEqual(true, MdSame1), 52 | ?assertEqual(true, MdSame2) 53 | end)}, 54 | {"indexes encode decode", 55 | ?_test(begin 56 | InputMD = dict:from_list([{?MD_INDEX, [{"index_bin", "foo"}, 57 | {"index_int", 10}]}]), 58 | ExpectedMD = [{?MD_INDEX, [{<<"index_bin">>, <<"foo">>}, 59 | {<<"index_int">>, <<"10">>}]}], 60 | Value = <<"test value">>, 61 | {OutputMD, _} = riak_pb_kv_codec:decode_content( 62 | riak_kv_pb:decode_msg( 63 | iolist_to_binary(riak_kv_pb:encode_msg( 64 | riak_pb_kv_codec:encode_content({InputMD, Value}))), 65 | rpbcontent)), 66 | ?assertEqual(ExpectedMD, dict:to_list(OutputMD)) 67 | end)}, 68 | {"empty content encode decode", 69 | ?_test(begin 70 | MetaData = dict:new(), 71 | Value = <<"test value">>, 72 | {MetaData2, Value2} = riak_pb_kv_codec:decode_content( 73 | riak_kv_pb:decode_msg( 74 | iolist_to_binary(riak_kv_pb:encode_msg( 75 | riak_pb_kv_codec:encode_content({MetaData, Value}))), 76 | rpbcontent)), 77 | ?assertEqual([], dict:to_list(MetaData2)), 78 | ?assertEqual(Value, Value2) 79 | end)}, 80 | {"empty repeated metas are removed/ignored", 81 | ?_test(begin 82 | MetaData = dict:from_list([{?MD_LINKS, []}, {?MD_USERMETA, []}, {?MD_INDEX, []}]), 83 | Value = <<"test value">>, 84 | {MetaData2, Value2} = riak_pb_kv_codec:decode_content( 85 | riak_kv_pb:decode_msg( 86 | iolist_to_binary(riak_kv_pb:encode_msg( 87 | riak_pb_kv_codec:encode_content({MetaData, Value}))), 88 | rpbcontent)), 89 | ?assertEqual([], dict:to_list(MetaData2)), 90 | ?assertEqual(Value, Value2) 91 | end)}, 92 | {"riak_dt-dtfetchreq-encode-decode", 93 | ?_test(begin 94 | RBin = <<10,1,97,18,1,97,26,1,97>>, 95 | R = riak_dt_pb:decode_msg(RBin, dtfetchreq), 96 | ?assertEqual(<<"a">>, R#dtfetchreq.type), 97 | ?assertEqual(<<"a">>, R#dtfetchreq.bucket), 98 | ?assertEqual(<<"a">>, R#dtfetchreq.key), 99 | ?assertEqual(undefined, R#dtfetchreq.r), 100 | ?assertEqual(undefined, R#dtfetchreq.pr), 101 | ?assertEqual(undefined, R#dtfetchreq.basic_quorum), 102 | ?assertEqual(undefined, R#dtfetchreq.notfound_ok), 103 | ?assertEqual(undefined, R#dtfetchreq.timeout), 104 | ?assertEqual(undefined, R#dtfetchreq.sloppy_quorum), 105 | ?assertEqual(undefined, R#dtfetchreq.n_val), 106 | ?assertEqual(true, R#dtfetchreq.include_context) 107 | end)}, 108 | {"msg code encode decode", 109 | ?_test(begin 110 | msg_code_encode_decode(0) 111 | end)} 112 | ]. 113 | 114 | msg_code_encode_decode(256) -> ok; 115 | msg_code_encode_decode(N) -> 116 | case riak_pb_codec:msg_type(N) of 117 | undefined -> 118 | ignore; 119 | MsgType -> 120 | ?assertEqual(N, riak_pb_codec:msg_code(MsgType)) 121 | end, 122 | msg_code_encode_decode(N+1). 123 | -------------------------------------------------------------------------------- /test/riak_pb_dt_codec_tests.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% test cases for riak_pb_dt_codec: Protocol Buffers utility functions for Riak DT types 4 | %% 5 | %% Copyright (c) 2013 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | -module(riak_pb_dt_codec_tests). 23 | 24 | -include_lib("eunit/include/eunit.hrl"). 25 | 26 | -include("riak_dt_pb.hrl"). 27 | 28 | -import(riak_pb_dt_codec, [decode_operation/1, 29 | operation_type/1, 30 | decode_fetch_response/1, 31 | encode_fetch_response/4, 32 | encode_update_request/4, 33 | decode_update_response/3 34 | ]). 35 | 36 | -define(CONTEXT, undefined_context). 37 | -define(SET_VALUE, [<<"binarytemple">>]). 38 | 39 | operation_type_gset_test() -> 40 | OpType = operation_type(#dtop{gset_op = #gsetop{}}), 41 | ?assertEqual(OpType, gset). 42 | 43 | decode_operation_gset_test() -> 44 | Op = #dtop{gset_op = #gsetop{adds = ?SET_VALUE}}, 45 | OpDecode = decode_operation(Op), 46 | ?assertEqual(OpDecode, {add_all, ?SET_VALUE}). 47 | 48 | decode_fetch_response_gset_test() -> 49 | Res = decode_fetch_response(#dtfetchresp{context = ?CONTEXT, type = 'GSET', value = #dtvalue{gset_value = ?SET_VALUE}}), 50 | ?assertEqual({gset, ?SET_VALUE, ?CONTEXT}, Res). 51 | 52 | decode_update_response_test() -> 53 | Res = decode_update_response( 54 | #dtupdateresp{set_value = ?SET_VALUE, context = ?CONTEXT}, set, true 55 | ), 56 | ?assertEqual({set, ?SET_VALUE, undefined_context}, Res). 57 | 58 | encode_fetch_response_gset_test() -> 59 | Resp = encode_fetch_response(gset, ?SET_VALUE, ?CONTEXT, []), 60 | ?assertMatch(#dtfetchresp{context= ?CONTEXT, type= 'GSET', value= #dtvalue{gset_value= ?SET_VALUE}}, Resp). 61 | 62 | encode_update_request_gset_test() -> 63 | Res = encode_update_request( 64 | {<<"btype">>, <<"bucket">>}, 65 | <<"key">>, 66 | {gset, {update, [{add_all, ?SET_VALUE}]}, ?CONTEXT}, 67 | [] 68 | ), 69 | ?assertMatch(#dtupdatereq{ 70 | bucket = <<"bucket">>, 71 | type = <<"btype">>, 72 | key = <<"key">>, 73 | op = #dtop{ 74 | gset_op = #gsetop{adds = ?SET_VALUE} 75 | } 76 | }, Res), 77 | ok 78 | . 79 | --------------------------------------------------------------------------------