├── .gitignore ├── .gitmodules ├── .hgignore ├── .hgtags ├── .travis.yml ├── LICENSE ├── Makefile ├── README.md ├── RELNOTES.md ├── doc └── overview.edoc ├── include ├── raw_http.hrl └── rhc.hrl ├── rebar ├── rebar.config ├── src ├── rhc.erl ├── rhc_bucket.erl ├── rhc_dt.erl ├── rhc_index.erl ├── rhc_listkeys.erl ├── rhc_mapred.erl ├── rhc_obj.erl ├── rhc_ts.erl └── riakhttpc.app.src └── tools.mk /.gitignore: -------------------------------------------------------------------------------- 1 | *.beam 2 | deps/* 3 | *~ 4 | .DS_Store 5 | *.orig 6 | doc/*.html 7 | doc/edoc-info 8 | doc/erlang.png 9 | doc/stylesheet.css 10 | .eunit/ 11 | ebin/ 12 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "tools"] 2 | path = tools 3 | url = https://github.com/basho/riak-client-tools.git 4 | -------------------------------------------------------------------------------- /.hgignore: -------------------------------------------------------------------------------- 1 | \.beam$ 2 | ^deps/.* 3 | ~$ 4 | ^\.DS_Store$ 5 | \.orig$ 6 | ^doc/.*\.html$ 7 | ^doc/edoc-info$ 8 | ^doc/erlang.png$ 9 | ^doc/stylesheet.css$ 10 | 11 | -------------------------------------------------------------------------------- /.hgtags: -------------------------------------------------------------------------------- 1 | 795d7ae531a75cab448755d1b0d9c55d802c82a9 riakhttpc-0.9.0 2 | e19c103d23633a690423337ad0bbe20e92297422 riakhttpc-0.9.1 3 | f62a90deeaf3bdd2f7cd1cac603893995ee87485 riakhttps-0.9.2 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: erlang 2 | otp_release: 3 | - R16B03 4 | - R15B03 5 | notifications: 6 | slack: 7 | secure: OIl/wglitmlcvzoHvu9hQCjfcAOQ57Kkxp/+e1IsbU4Oz6Q3NHy3j+fsBAkP/KWWDM03c/0Onqsiua76sztfadoJXpZQsbiMLwUPR2clMgGOY3HIrPF4XMzT72tXQRl8/M1o885X+Eejdc2DE3F4h/iVyS45GYPfBXtnzla+hRg= 8 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | riakhttpc: Riak HTTP Client 2 | 3 | Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved. 4 | 5 | This file is provided to you under the Apache License, 6 | Version 2.0 (the "License"); you may not use this file 7 | except in compliance with the License. You may obtain 8 | a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, 13 | software distributed under the License is distributed on an 14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | KIND, either express or implied. See the License for the 16 | specific language governing permissions and limitations 17 | under the License. 18 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: all lint clean compile deps distclean release docs 2 | 3 | PROJDIR := $(realpath $(CURDIR)) 4 | REBAR := $(PROJDIR)/rebar 5 | 6 | all: deps compile 7 | 8 | lint: xref dialyzer 9 | 10 | compile: deps 11 | $(REBAR) compile 12 | 13 | deps: 14 | $(REBAR) get-deps 15 | 16 | clean: 17 | $(REBAR) clean 18 | 19 | distclean: clean 20 | $(REBAR) delete-deps 21 | 22 | release: compile 23 | ifeq ($(VERSION),) 24 | $(error VERSION must be set to build a release and deploy this package) 25 | endif 26 | ifeq ($(RELEASE_GPG_KEYNAME),) 27 | $(error RELEASE_GPG_KEYNAME must be set to build a release and deploy this package) 28 | endif 29 | @echo "==> Tagging version $(VERSION)" 30 | @./tools/build/publish $(VERSION) master validate 31 | @git tag --sign -a "$(VERSION)" -m "riak-erlang-http-client $(VERSION)" --local-user "$(RELEASE_GPG_KEYNAME)" 32 | @git push --tags 33 | @./tools/build/publish $(VERSION) master 'Riak Erlang HTTP Client' 'riak-erlang-http-client' 34 | 35 | DIALYZER_APPS = kernel stdlib crypto ibrowse 36 | 37 | include tools.mk 38 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # riak-erlang-http-client 2 | 3 | ## Build Status 4 | 5 | [![Build Status](https://travis-ci.org/basho/riak-erlang-http-client.svg?branch=develop)](https://travis-ci.org/basho/riak-erlang-http-client) 6 | 7 | `riak-erlang-http-client` is an Erlang client for Riak, using the HTTP interface 8 | 9 | ## Quick Start 10 | 11 | You must have [Erlang/OTP R15B01](http://erlang.org/download.html) or later and a GNU-style build system to compile and run `riak-erlang-http-client`. 12 | 13 | ```sh 14 | git clone git://github.com/basho/riak-erlang-http-client.git 15 | cd riak-erlang-http-client 16 | make 17 | ``` 18 | 19 | If the Protocol Buffers Riak Erlang Client ([riak-erlang-client](http://github.com/basho/riak-erlang-client)) is already familiar to you, you should find this client familiar. Just substitute calls to `riakc_pb_socket` with calls to `rhc`. 20 | 21 | As a quick example, here is how to create and retrieve a value with the key "foo" in the bucket "bar" using this client. 22 | 23 | First, start up an Erlang shell with the path to `riak-erlang-http-client` and all dependencies included, then start `sasl` and `ibrowse`: 24 | 25 | ```sh 26 | erl -pa path/to/riak-erlang-http-client/ebin path/to/riak-erlang-http-client/deps/*/ebin 27 | Eshell V5.8.2 (abort with ^G) 28 | 1> [ ok = application:start(A) || A <- [sasl, ibrowse] ]. 29 | ``` 30 | 31 | Next, create your client: 32 | 33 | ```erlang 34 | 2> IP = "127.0.0.1", 35 | 2> Port = 8098, 36 | 2> Prefix = "riak", 37 | 2> Options = [], 38 | 2> C = rhc:create(IP, Port, Prefix, Options). 39 | {rhc,"10.0.0.42",80,"riak",[{client_id,"ACoc4A=="}]} 40 | ``` 41 | 42 | Sidenote: if you will be using the defaults, as in the example above, you may call `rhc:create/0` instead of specifying the defaults yourself. 43 | 44 | Create a new object, and store it with `rhc:put/2`: 45 | 46 | ```erlang 47 | 3> Bucket = <<"bar">>, 48 | 3> Key = <<"foo">>, 49 | 3> Data = <<"hello world">>, 50 | 3> ContentType = <<"text/plain">>, 51 | 3> Object0 = riakc_obj:new(Bucket, Key, Data, ContentType), 52 | 3> rhc:put(C, Object0). 53 | ok 54 | ``` 55 | 56 | Retrieve an object with `rhc:get/3`: 57 | 58 | ```erlang 59 | 4> Bucket = <<"bar">>, 60 | 4> Key = <<"foo">>, 61 | 4> {ok, Object1} = rhc:get(C, Bucket, Key). 62 | {ok,{riakc_obj,<<"bar">>,<<"foo">>, 63 | <<107,206,97,96,96,96,204,96,202,5,82,44,12,167,92,95,100, 64 | 48,37,50,230,177,50,...>>, 65 | [{{dict,3,16,16,8,80,48, 66 | {[],[],[],[],[],[],[],[],[],[],[],[],...}, 67 | {{[],[],[],[],[],[],[],[],[],[],...}}}, 68 | <<"hello world">>}], 69 | undefined,undefined}} 70 | ``` 71 | 72 | Please refer to the generated documentation for more information: 73 | 74 | ```sh 75 | make doc && open doc/index.html 76 | ``` 77 | 78 | ## Contributing 79 | 80 | We encourage contributions to `riak-erlang-http-client` from the community. 81 | 82 | * Fork the `riak-erlang-http-client` repository on [GitHub](https://github.com/basho/riak-erlang-http-client) 83 | 84 | * Clone your fork or add the remote if you already have a clone of the repository. 85 | 86 | git clone git@github.com:yourusername/riak-erlang-http-client.git 87 | # or 88 | git remote add mine git@github.com:yourusername/riak-erlang-http-client.git 89 | 90 | * Create a topic branch for your change. 91 | 92 | git checkout -b some-topic-branch 93 | 94 | * Make your change and commit. Use a clear and descriptive commit message, spanning multiple lines if detailed explanation is needed. 95 | 96 | * Push to your fork of the repository and then send a pull-request through Github. 97 | 98 | git push mine some-topic-branch 99 | 100 | * A Basho engineer or community maintainer will review your patch and merge it into the main repository or send you feedback. 101 | -------------------------------------------------------------------------------- /RELNOTES.md: -------------------------------------------------------------------------------- 1 | Release Notes 2 | ============= 3 | 4 | * [`2.2.0`](https://github.com/basho/riak-erlang-http-client/issues?q=milestone%3Ariak-erlang-http-client-2.2.0) 5 | -------------------------------------------------------------------------------- /doc/overview.edoc: -------------------------------------------------------------------------------- 1 | @title The Riak Erlang HTTP Client (riakhttpc) 2 | @copyright 2010 Basho Technologies 3 | 4 | @doc The riakhttpc application includes modules that provide access to 5 | Riak's HTTP interface. Included modules are: 6 | 7 |
8 |
{@link rhc}
9 |
The `rhc' module is the main interface for interacting 10 | with this application. It includes functions for creating clients, 11 | and for performing gets, puts, map/reduces, and so on, using those 12 | clients. This should generally be the only module you need to learn 13 | if you're not hacking on this application.
14 | 15 |
{@link rhc_obj}
16 |
The `rhc_obj' module provides utilities for transforming 17 | HTTP responses into `riakc_obj' objects, and for transforming 18 | those objects back into HTTP requests. More information about 19 | `riakc_obj' objects is available in the proctol-buffers-based 20 | 21 | riak-erlang-client project.
22 | 23 |
{@link rhc_bucket}
24 |
The `rhc_bucket' module provides utilities for 25 | translating bucket-level requests back and forth between HTTP and 26 | Erlang formats.
27 | 28 |
{@link rhc_listkeys}
29 |
The `rhc_listkeys' module handles decoding of key list 30 | responses.
31 | 32 |
{@link rhc_mapred}
33 |
The `rhc_mapred' module handles decoding of map/reduce 34 | results.
35 |
36 | 37 | It is a goal of `riakhttpc' to provide API-compatible 38 | functionality with 39 | riakc, 40 | the protocol buffers Riak Erlang client. That is, the functions of 41 | the {@link rhc} module should take the same parameters and provide the 42 | same results as those sharing their name in the 43 | riak_pb_socket 44 | module. So, a call like: 45 | 46 | ``` 47 | rhc:get(Client, <<"mybucket">>, <<"mykey">>). 48 | ''' 49 | 50 | should give the same results as 51 | 52 | ``` 53 | riakc_pb_socket:get(Client, <<"mybucket">>, <<"mykey">>). 54 | ''' 55 | 56 |

Basic Usage

57 | 58 | The basic pattern of use of the Riak Erlang HTTP client is: 59 | 60 |

1. Create a client

61 | 62 | If you're connecting to the default Riak HTTP port on the local 63 | machine, creating a client is easy: 64 | 65 | ``` 66 | C = rhc:create(). 67 | ''' 68 | 69 | To connect to a remote machine, or a non-standard port: 70 | 71 | ``` 72 | IP = "10.0.0.42", 73 | Port = 80, 74 | Prefix = "riak", 75 | Options = [] 76 | C = rhc:create(IP, Port, Prefix, Options). 77 | ''' 78 | 79 | Test the connection with {@link rhc:ping/1}: 80 | 81 | ``` 82 | ok = rhc:ping(C). 83 | ''' 84 | 85 |

2. Make a new object

86 | 87 | Make new objects using the {@link //riakc/riakc_obj} module: 88 | 89 | ``` 90 | O = riakc_obj:new(<<"bucket">>, <<"key">>, 91 | <<"here is some data to store">>, 92 | <<"text/plain">>). 93 | ''' 94 | 95 |

3. Retrieving an object

96 | 97 | To read an object from Read, use {@link rhc:get/2}: 98 | 99 | ``` 100 | {ok, O1} = rhc:get(C, <<"bucket">>, <<"key">>). 101 | ''' 102 | 103 |

4. Modifying an object

104 | 105 | To change an object's value, use {@link //riakc/riakc_obj:update_value/2}: 106 | 107 | ``` 108 | O2 = riakc_obj:update_value(O1, <<"my new data">>). 109 | ''' 110 | 111 |

5. Storing an object

112 | 113 | To store an object to Riak, use {@link rhc:put/2}: 114 | 115 | ``` 116 | ok = rhc:put(C, O2). 117 | ''' 118 | 119 | 120 |

6. Map/reduce

121 | 122 | Execute map/reduce queries with {@link rhc:mapred/3}: 123 | 124 | ``` 125 | Q = [{map, {jsanon, <<"function(v) { return [v]; }">>}, <<>>, true}], 126 | {ok, Results} = rhc:mapred(C, [{<<"bucket">>, <<"key">>}], Q). 127 | ''' 128 | 129 | More information about the options that can be passed to each of these 130 | functions, and also about the streaming versions of map/reduce 131 | queries, can be found in the documentation of the {@link rhc} module. 132 | -------------------------------------------------------------------------------- /include/raw_http.hrl: -------------------------------------------------------------------------------- 1 | %% This file is provided to you under the Apache License, 2 | %% Version 2.0 (the "License"); you may not use this file 3 | %% except in compliance with the License. You may obtain 4 | %% a copy of the License at 5 | 6 | %% http://www.apache.org/licenses/LICENSE-2.0 7 | 8 | %% Unless required by applicable law or agreed to in writing, 9 | %% software distributed under the License is distributed on an 10 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 11 | %% KIND, either express or implied. See the License for the 12 | %% specific language governing permissions and limitations 13 | %% under the License. 14 | 15 | %% Constants used by the raw_http resources 16 | %% original source at 17 | %% http://github.com/basho/riak_kv/blob/master/src/riak_kv_wm_raw.hrl 18 | 19 | %%====================================================================== 20 | %% Names of riak_object metadata fields 21 | %%====================================================================== 22 | -define(MD_CTYPE, <<"content-type">>). 23 | -define(MD_CHARSET, <<"charset">>). 24 | -define(MD_ENCODING, <<"content-encoding">>). 25 | -define(MD_VTAG, <<"X-Riak-VTag">>). 26 | -define(MD_LINKS, <<"Links">>). 27 | -define(MD_LASTMOD, <<"X-Riak-Last-Modified">>). 28 | -define(MD_USERMETA, <<"X-Riak-Meta">>). 29 | -define(MD_INDEX, <<"index">>). 30 | 31 | %%====================================================================== 32 | %% Names of HTTP header fields 33 | %%====================================================================== 34 | -define(HEAD_CTYPE, "Content-Type"). 35 | -define(HEAD_VCLOCK, "X-Riak-Vclock"). 36 | -define(HEAD_LINK, "Link"). 37 | -define(HEAD_ENCODING, "Content-Encoding"). 38 | -define(HEAD_CLIENT, "X-Riak-ClientId"). 39 | -define(HEAD_USERMETA_PREFIX, "x-riak-meta-"). 40 | -define(HEAD_INDEX_PREFIX, "X-Riak-Index-"). 41 | 42 | %%====================================================================== 43 | %% JSON keys/values 44 | %%====================================================================== 45 | 46 | %% Top-level keys in JSON bucket responses 47 | -define(JSON_PROPS, <<"props">>). 48 | -define(JSON_KEYS, <<"keys">>). 49 | 50 | %% Names of JSON fields in bucket properties 51 | -define(JSON_ALLOW_MULT, <<"allow_mult">>). 52 | -define(JSON_BACKEND, <<"backend">>). 53 | -define(JSON_BASIC_Q, <<"basic_quorum">>). 54 | -define(JSON_BIG_VC, <<"big_vclock">>). 55 | -define(JSON_CHASH, <<"chash_keyfun">>). 56 | -define(JSON_DW, <<"dw">>). 57 | -define(JSON_LINKFUN, <<"linkfun">>). 58 | -define(JSON_LWW, <<"last_write_wins">>). 59 | -define(JSON_NF_OK, <<"notfound_ok">>). 60 | -define(JSON_N_VAL, <<"n_val">>). 61 | -define(JSON_OLD_VC, <<"old_vclock">>). 62 | -define(JSON_POSTCOMMIT, <<"postcommit">>). 63 | -define(JSON_PR, <<"pr">>). 64 | -define(JSON_PRECOMMIT, <<"precommit">>). 65 | -define(JSON_PW, <<"pw">>). 66 | -define(JSON_R, <<"r">>). 67 | -define(JSON_REPL, <<"repl">>). 68 | -define(JSON_RW, <<"rw">>). 69 | -define(JSON_SEARCH, <<"search">>). 70 | -define(JSON_SMALL_VC, <<"small_vclock">>). 71 | -define(JSON_W, <<"w">>). 72 | -define(JSON_YOUNG_VC, <<"young_vclock">>). 73 | 74 | %% Valid quorum property values in JSON 75 | -define(JSON_ALL, <<"all">>). 76 | -define(JSON_ONE, <<"one">>). 77 | -define(JSON_QUORUM, <<"quorum">>). 78 | 79 | %% Valid 'repl' property values in JSON 80 | -define(JSON_BOTH, <<"both">>). 81 | -define(JSON_FULLSYNC, <<"fullsync">>). 82 | -define(JSON_REALTIME, <<"realtime">>). 83 | 84 | %% MapReduce JSON fields 85 | -define(JSON_MOD, <<"mod">>). %% also used by bucket props 86 | -define(JSON_FUN, <<"fun">>). %% also used by bucket props 87 | -define(JSON_JSANON, <<"jsanon">>). 88 | -define(JSON_JSBUCKET, <<"bucket">>). 89 | -define(JSON_JSFUN, <<"jsfun">>). 90 | -define(JSON_JSKEY, <<"key">>). 91 | 92 | %% 2i fields 93 | -define(JSON_RESULTS, <<"results">>). 94 | -define(JSON_CONTINUATION, <<"continuation">>). 95 | 96 | %% dt related-fields 97 | -define(JSON_HLL_PRECISION, <<"hll_precision">>). 98 | 99 | %%====================================================================== 100 | %% Names of HTTP query parameters 101 | %%====================================================================== 102 | -define(Q_PROPS, "props"). 103 | -define(Q_KEYS, "keys"). 104 | -define(Q_BUCKETS, "buckets"). 105 | -define(Q_FALSE, "false"). 106 | -define(Q_TRUE, "true"). 107 | -define(Q_TIMEOUT, "timeout"). 108 | -define(Q_STREAM, "stream"). 109 | -define(Q_VTAG, "vtag"). 110 | -define(Q_RETURNBODY, "returnbody"). 111 | -define(Q_RETURNTERMS, "return_terms"). 112 | -define(Q_PAGINATION_SORT, "pagination_sort"). 113 | -define(Q_MAXRESULTS, "max_results"). 114 | -define(Q_CONTINUATION, "continuation"). 115 | -define(Q_TERM_REGEX, "term_regex"). 116 | -------------------------------------------------------------------------------- /include/rhc.hrl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riakhttpc: Riak HTTP Client 4 | %% 5 | %% Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | -define(DEFAULT_TIMEOUT, 60000). 24 | 25 | -record(rhc, {ip, 26 | port, 27 | prefix, 28 | options}). 29 | -------------------------------------------------------------------------------- /rebar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/basho/riak-erlang-http-client/2d4e58371e4d982bee1e0ad804b1403b1333573c/rebar -------------------------------------------------------------------------------- /rebar.config: -------------------------------------------------------------------------------- 1 | %% -*- mode: Erlang -*- 2 | 3 | {erl_opts, []}. 4 | 5 | {deps, 6 | [ 7 | %% ibrowse for doing HTTP requests 8 | {ibrowse, "4.3", {git, "https://github.com/basho/ibrowse.git", {tag, "v4.3"}}}, 9 | %% webmachine for multipart content parsing, will pull in mochiweb as a dep 10 | {webmachine, "1.10.8", {git, "https://github.com/basho/webmachine", {tag, "1.10.8p2"}}}, 11 | %% riak-erlang-client for riakc_obj 12 | {riakc, "2.5.*", {git, "https://github.com/basho/riak-erlang-client", {tag, "2.5.1"}}} 13 | ]}. 14 | 15 | {edoc_opts, 16 | [ 17 | %% handle macro expansion in edoc 18 | {preprocess, true}, 19 | {include, "./deps/riakc/include/riakc.hrl"} 20 | ]}. 21 | 22 | {clean_files, ["doc/*.html", "doc/*.png", "doc/edoc-info", "doc/*.css"]}. 23 | -------------------------------------------------------------------------------- /src/rhc.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riakhttpc: Riak HTTP Client 4 | %% 5 | %% Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | %% @doc Riak Erlang HTTP Client. This module provides access to Riak's 24 | %% HTTP interface. For basic usage, please read 25 | %% the riakhttpc application overview. 26 | -module(rhc). 27 | 28 | -export([create/0, create/4, 29 | ip/1, 30 | port/1, 31 | prefix/1, 32 | options/1, 33 | ping/1, 34 | get_client_id/1, 35 | get_server_info/1, 36 | get_server_stats/1, 37 | get/3, get/4, 38 | put/2, put/3, 39 | delete/3, delete/4, delete_obj/2, delete_obj/3, 40 | list_buckets/1, 41 | list_buckets/2, 42 | stream_list_buckets/1, 43 | stream_list_buckets/2, 44 | list_keys/2, 45 | list_keys/3, 46 | stream_list_keys/2, 47 | stream_list_keys/3, 48 | get_index/4, get_index/5, 49 | stream_index/4, stream_index/5, 50 | get_bucket/2, 51 | set_bucket/3, 52 | reset_bucket/2, 53 | get_bucket_type/2, 54 | set_bucket_type/3, 55 | mapred/3,mapred/4, 56 | mapred_stream/4, mapred_stream/5, 57 | mapred_bucket/3, mapred_bucket/4, 58 | mapred_bucket_stream/5, 59 | search/3, search/5, 60 | counter_incr/4, counter_incr/5, 61 | counter_val/3, counter_val/4, 62 | fetch_type/3, fetch_type/4, 63 | update_type/4, update_type/5, 64 | modify_type/5, 65 | get_preflist/3 66 | ]). 67 | 68 | -include("raw_http.hrl"). 69 | -include("rhc.hrl"). 70 | -include_lib("riakc/include/riakc.hrl"). 71 | 72 | -ifdef(TEST). 73 | -include_lib("eunit/include/eunit.hrl"). 74 | -endif. 75 | 76 | -export_type([rhc/0]). 77 | -opaque rhc() :: #rhc{}. 78 | 79 | %% @doc Create a client for connecting to the default port on localhost. 80 | %% @equiv create("127.0.0.1", 8098, "riak", []) 81 | create() -> 82 | create("127.0.0.1", 8098, "riak", []). 83 | 84 | %% @doc Create a client for connecting to a Riak node. 85 | %% 86 | %% Connections are made to: 87 | %% ```http://IP:Port/Prefix/(/)''' 88 | %% 89 | %% Defaults for r, w, dw, rw, and return_body may be passed in 90 | %% the Options list. The client id can also be specified by 91 | %% adding `{client_id, ID}' to the Options list. 92 | %% @spec create(string(), integer(), string(), Options::list()) -> rhc() 93 | create(IP, Port, Prefix, Opts0) when is_list(IP), is_integer(Port), 94 | is_list(Prefix), is_list(Opts0) -> 95 | Opts = case proplists:lookup(client_id, Opts0) of 96 | none -> [{client_id, random_client_id()}|Opts0]; 97 | Bin when is_binary(Bin) -> 98 | [{client_id, binary_to_list(Bin)} 99 | | [ O || O={K,_} <- Opts0, K =/= client_id ]]; 100 | _ -> 101 | Opts0 102 | end, 103 | #rhc{ip=IP, port=Port, prefix=Prefix, options=Opts}. 104 | 105 | %% @doc Get the IP this client will connect to. 106 | %% @spec ip(rhc()) -> string() 107 | ip(#rhc{ip=IP}) -> IP. 108 | 109 | %% @doc Get the Port this client will connect to. 110 | %% @spec port(rhc()) -> integer() 111 | port(#rhc{port=Port}) -> Port. 112 | 113 | %% @doc Get the prefix this client will use for object URLs 114 | %% @spec prefix(rhc()) -> string() 115 | prefix(#rhc{prefix=Prefix}) -> Prefix. 116 | 117 | %% @doc Ping the server by requesting the "/ping" resource. 118 | %% @spec ping(rhc()) -> ok|{error, term()} 119 | ping(Rhc) -> 120 | Url = ping_url(Rhc), 121 | case request(get, Url, ["200","204"], [], [], Rhc) of 122 | {ok, _Status, _Headers, _Body} -> 123 | ok; 124 | {error, Error} -> 125 | {error, Error} 126 | end. 127 | 128 | %% @doc Get the client ID that this client will use when storing objects. 129 | %% @spec get_client_id(rhc()) -> {ok, string()} 130 | get_client_id(Rhc) -> 131 | {ok, client_id(Rhc, [])}. 132 | 133 | %% @doc Get some basic information about the server. The proplist returned 134 | %% should include `node' and `server_version' entries. 135 | %% @spec get_server_info(rhc()) -> {ok, proplist()}|{error, term()} 136 | get_server_info(Rhc) -> 137 | Url = stats_url(Rhc), 138 | case request(get, Url, ["200"], [], [], Rhc) of 139 | {ok, _Status, _Headers, Body} -> 140 | {struct, Response} = mochijson2:decode(Body), 141 | {ok, erlify_server_info(Response)}; 142 | {error, Error} -> 143 | {error, Error} 144 | end. 145 | 146 | %% @doc Get the list of full stats from a /stats call to the server. 147 | %% @spec get_server_stats(rhc()) -> {ok, proplist()}|{error, term()} 148 | get_server_stats(Rhc) -> 149 | Url = stats_url(Rhc), 150 | case request(get, Url, ["200"], [], [], Rhc) of 151 | {ok, _Status, _Headers, Body} -> 152 | {struct, Response} = mochijson2:decode(Body), 153 | Stats = lists:flatten(Response), 154 | {ok, Stats}; 155 | {error, Error} -> 156 | {error, Error} 157 | end. 158 | 159 | %% @equiv get(Rhc, Bucket, Key, []) 160 | get(Rhc, Bucket, Key) -> 161 | get(Rhc, Bucket, Key, []). 162 | 163 | %% @doc Get the objects stored under the given bucket and key. 164 | %% 165 | %% Allowed options are: 166 | %%
167 | %%
`r'
168 | %%
The 'R' value to use for the read
169 | %%
`timeout'
170 | %%
The server-side timeout for the write in ms
171 | %%
172 | %% 173 | %% The term in the second position of the error tuple will be 174 | %% `notfound' if the key was not found. 175 | %% @spec get(rhc(), bucket(), key(), proplist()) 176 | %% -> {ok, riakc_obj()}|{error, term()} 177 | get(Rhc, Bucket, Key, Options) -> 178 | Qs = get_q_params(Rhc, Options), 179 | Url = make_url(Rhc, Bucket, Key, Qs), 180 | case request(get, Url, ["200", "300"], [], [], Rhc) of 181 | {ok, _Status, Headers, Body} -> 182 | {ok, rhc_obj:make_riakc_obj(Bucket, Key, Headers, Body)}; 183 | {error, {ok, "404", Headers, _}} -> 184 | case proplists:get_value(deletedvclock, Options) of 185 | true -> 186 | case proplists:get_value("X-Riak-Vclock", Headers) of 187 | undefined -> 188 | {error, notfound}; 189 | Vclock -> 190 | {error, {notfound, base64:decode(Vclock)}} 191 | end; 192 | _ -> 193 | {error, notfound} 194 | end; 195 | {error, Error} -> 196 | {error, Error} 197 | end. 198 | 199 | %% @equiv put(Rhc, Object, []) 200 | put(Rhc, Object) -> 201 | put(Rhc, Object, []). 202 | 203 | %% @doc Store the given object in Riak. 204 | %% 205 | %% Allowed options are: 206 | %%
207 | %%
`w'
208 | %%
The 'W' value to use for the write
209 | %%
`dw'
210 | %%
The 'DW' value to use for the write
211 | %%
`timeout'
212 | %%
The server-side timeout for the write in ms
213 | %%
return_body
214 | %%
Whether or not to return the updated object in the 215 | %% response. `ok' is returned if return_body is false. 216 | %% `{ok, Object}' is returned if return_body is true.
217 | %%
218 | %% @spec put(rhc(), riakc_obj(), proplist()) 219 | %% -> ok|{ok, riakc_obj()}|{error, term()} 220 | put(Rhc, Object, Options) -> 221 | Qs = put_q_params(Rhc, Options), 222 | Bucket = riakc_obj:bucket(Object), 223 | Key = riakc_obj:key(Object), 224 | Url = make_url(Rhc, Bucket, Key, Qs), 225 | Method = if Key =:= undefined -> post; 226 | true -> put 227 | end, 228 | {Headers0, Body} = rhc_obj:serialize_riakc_obj(Rhc, Object), 229 | Headers = [{?HEAD_CLIENT, client_id(Rhc, Options)} 230 | |Headers0], 231 | case request(Method, Url, ["200", "204", "300"], Headers, Body, Rhc) of 232 | {ok, Status, ReplyHeaders, ReplyBody} -> 233 | if Status =:= "204" -> 234 | ok; 235 | true -> 236 | {ok, rhc_obj:make_riakc_obj(Bucket, Key, 237 | ReplyHeaders, ReplyBody)} 238 | end; 239 | {error, Error} -> 240 | {error, Error} 241 | end. 242 | 243 | %% @doc Increment the counter stored under `bucket', `key' 244 | %% by the given `amount'. 245 | %% @equiv counter_incr(Rhc, Bucket, Key, Amt, []) 246 | -spec counter_incr(rhc(), binary(), binary(), integer()) -> ok | {ok, integer()} 247 | | {error, term()}. 248 | counter_incr(Rhc, Bucket, Key, Amt) -> 249 | counter_incr(Rhc, Bucket, Key, Amt, []). 250 | 251 | %% @doc Increment the counter stored at `bucket', `key' by 252 | %% `Amt'. Note: `Amt' can be a negative or positive integer. 253 | %% 254 | %% Allowed options are: 255 | %%
256 | %%
`w'
257 | %%
The 'W' value to use for the write
258 | %%
`dw'
259 | %%
The 'DW' value to use for the write
260 | %%
`pw'
261 | %%
The 'PW' value to use for the write
262 | %%
`timeout'
263 | %%
The server-side timeout for the write in ms
264 | %%
`returnvalue'
265 | %%
Whether or not to return the updated value in the 266 | %% response. `ok' is returned if returnvalue is absent | `false'. 267 | %% `{ok, integer()}' is returned if returnvalue is `true'.
268 | %%
269 | %% See the riak docs at http://docs.basho.com/riak/latest/references/apis/http/ for details 270 | -spec counter_incr(rhc(), binary(), binary(), integer(), list()) -> ok | {ok, integer()} 271 | | {error, term()}. 272 | counter_incr(Rhc, Bucket, Key, Amt, Options) -> 273 | Qs = counter_q_params(Rhc, Options), 274 | Url = make_counter_url(Rhc, Bucket, Key, Qs), 275 | Body = integer_to_list(Amt), 276 | case request(post, Url, ["200", "204"], [], Body, Rhc) of 277 | {ok, Status, _ReplyHeaders, ReplyBody} -> 278 | if Status =:= "204" -> 279 | ok; 280 | true -> 281 | {ok, list_to_integer(binary_to_list(ReplyBody))} 282 | end; 283 | {error, Error} -> 284 | {error, Error} 285 | end. 286 | 287 | %% @doc Get the counter stored at `bucket', `key'. 288 | -spec counter_val(rhc(), term(), term()) -> {ok, integer()} | {error, term()}. 289 | counter_val(Rhc, Bucket, Key) -> 290 | counter_val(Rhc, Bucket, Key, []). 291 | 292 | %% @doc Get the counter stored at `bucket', `key'. 293 | %% 294 | %% Allowed options are: 295 | %%
296 | %%
`r'
297 | %%
The 'R' value to use for the read
298 | %%
`pr'
299 | %%
The 'PR' value to use for the read
300 | %%
`pw'
301 | %%
The 'PW' value to use for the write
302 | %%
`timeout'
303 | %%
The server-side timeout for the write in ms
304 | %%
`notfound_ok'
305 | %%
if `true' not_found replies from vnodes count toward read quorum.
306 | %%
`basic_quorum'
307 | %%
When set to `true' riak will return a value as soon as it gets a quorum of responses.
308 | %%
309 | %% See the riak docs at http://docs.basho.com/riak/latest/references/apis/http/ for details 310 | -spec counter_val(rhc(), term(), term(), list()) -> {ok, integer()} | {error, term()}. 311 | counter_val(Rhc, Bucket, Key, Options) -> 312 | Qs = counter_q_params(Rhc, Options), 313 | Url = make_counter_url(Rhc, Bucket, Key, Qs), 314 | case request(get, Url, ["200"], [], [], Rhc) of 315 | {ok, "200", _ReplyHeaders, ReplyBody} -> 316 | {ok, list_to_integer(binary_to_list(ReplyBody))}; 317 | {error, Error} -> 318 | {error, Error} 319 | end. 320 | 321 | %% @equiv delete(Rhc, Bucket, Key, []) 322 | delete(Rhc, Bucket, Key) -> 323 | delete(Rhc, Bucket, Key, []). 324 | 325 | %% @doc Delete the given key from the given bucket. 326 | %% 327 | %% Allowed options are: 328 | %%
329 | %%
`rw'
330 | %%
The 'RW' value to use for the delete
331 | %%
`timeout'
332 | %%
The server-side timeout for the write in ms
333 | %%
334 | %% @spec delete(rhc(), bucket(), key(), proplist()) -> ok|{error, term()} 335 | delete(Rhc, Bucket, Key, Options) -> 336 | Qs = delete_q_params(Rhc, Options), 337 | Url = make_url(Rhc, Bucket, Key, Qs), 338 | Headers0 = case lists:keyfind(vclock, 1, Options) of 339 | false -> []; 340 | {vclock, V} -> 341 | [{?HEAD_VCLOCK, base64:encode_to_string(V)}] 342 | end, 343 | Headers = [{?HEAD_CLIENT, client_id(Rhc, Options)}|Headers0], 344 | case request(delete, Url, ["204"], Headers, [], Rhc) of 345 | {ok, "204", _Headers, _Body} -> ok; 346 | {error, Error} -> {error, Error} 347 | end. 348 | 349 | 350 | %% @equiv delete_obj(Rhc, Obj, []) 351 | delete_obj(Rhc, Obj) -> 352 | delete_obj(Rhc, Obj, []). 353 | 354 | %% @doc Delete the key of the given object, using the contained vector 355 | %% clock if present. 356 | %% @equiv delete(Rhc, riakc_obj:bucket(Obj), riakc_obj:key(Obj), [{vclock, riakc_obj:vclock(Obj)}|Options]) 357 | delete_obj(Rhc, Obj, Options) -> 358 | Bucket = riakc_obj:bucket(Obj), 359 | Key = riakc_obj:key(Obj), 360 | VClock = riakc_obj:vclock(Obj), 361 | delete(Rhc, Bucket, Key, [{vclock, VClock}|Options]). 362 | 363 | list_buckets(Rhc) -> 364 | list_buckets(Rhc, undefined). 365 | 366 | list_buckets(Rhc, BucketType) when is_binary(BucketType) -> 367 | list_buckets(Rhc, BucketType, undefined); 368 | list_buckets(Rhc, Timeout) -> 369 | list_buckets(Rhc, undefined, Timeout). 370 | 371 | list_buckets(Rhc, BucketType, Timeout) -> 372 | {ok, ReqId} = stream_list_buckets(Rhc, BucketType, Timeout), 373 | rhc_listkeys:wait_for_list(ReqId, Timeout). 374 | 375 | stream_list_buckets(Rhc) -> 376 | stream_list_buckets(Rhc, undefined). 377 | 378 | stream_list_buckets(Rhc, BucketType) when is_binary(BucketType) -> 379 | stream_list_buckets(Rhc, BucketType, undefined); 380 | stream_list_buckets(Rhc, Timeout) -> 381 | stream_list_buckets(Rhc, undefined, Timeout). 382 | 383 | stream_list_buckets(Rhc, BucketType, Timeout) -> 384 | ParamList0 = [{?Q_BUCKETS, ?Q_STREAM}, 385 | {?Q_PROPS, ?Q_FALSE}], 386 | ParamList = 387 | case Timeout of 388 | undefined -> ParamList0; 389 | N -> ParamList0 ++ [{?Q_TIMEOUT, N}] 390 | end, 391 | Url = make_url(Rhc, {BucketType, undefined}, undefined, ParamList), 392 | StartRef = make_ref(), 393 | Pid = spawn(rhc_listkeys, list_acceptor, [self(), StartRef, buckets]), 394 | case request_stream(Pid, get, Url, [], [], Rhc) of 395 | {ok, ReqId} -> 396 | Pid ! {ibrowse_req_id, StartRef, ReqId}, 397 | {ok, StartRef}; 398 | {error, Error} -> {error, Error} 399 | end. 400 | 401 | list_keys(Rhc, Bucket) -> 402 | list_keys(Rhc, Bucket, undefined). 403 | 404 | %% @doc List the keys in the given bucket. 405 | %% @spec list_keys(rhc(), bucket(), integer()) -> {ok, [key()]}|{error, term()} 406 | 407 | list_keys(Rhc, Bucket, Timeout) -> 408 | {ok, ReqId} = stream_list_keys(Rhc, Bucket, Timeout), 409 | rhc_listkeys:wait_for_list(ReqId, ?DEFAULT_TIMEOUT). 410 | 411 | stream_list_keys(Rhc, Bucket) -> 412 | stream_list_keys(Rhc, Bucket, undefined). 413 | 414 | %% @doc Stream key lists to a Pid. Messages sent to the Pid will 415 | %% be of the form `{reference(), message()}' 416 | %% where `message()' is one of: 417 | %%
418 | %%
`done'
419 | %%
end of key list, no more messages will be sent
420 | %%
`{keys, [key()]}'
421 | %%
a portion of the key list
422 | %%
`{error, term()}'
423 | %%
an error occurred
424 | %%
425 | %% @spec stream_list_keys(rhc(), bucket(), integer()) -> 426 | %% {ok, reference()}|{error, term()} 427 | stream_list_keys(Rhc, Bucket, Timeout) -> 428 | ParamList0 = [{?Q_KEYS, ?Q_STREAM}, 429 | {?Q_PROPS, ?Q_FALSE}], 430 | ParamList = 431 | case Timeout of 432 | undefined -> ParamList0; 433 | N -> ParamList0 ++ [{?Q_TIMEOUT, N}] 434 | end, 435 | Url = make_url(Rhc, Bucket, undefined, ParamList), 436 | StartRef = make_ref(), 437 | Pid = spawn(rhc_listkeys, list_acceptor, [self(), StartRef, keys]), 438 | case request_stream(Pid, get, Url, [], [], Rhc) of 439 | {ok, ReqId} -> 440 | Pid ! {ibrowse_req_id, StartRef, ReqId}, 441 | {ok, StartRef}; 442 | {error, Error} -> {error, Error} 443 | end. 444 | 445 | %% @doc Query a secondary index. 446 | %% @spec get_index(rhc(), bucket(), index(), index_query()) -> 447 | %% {ok, index_results()} | {error, term()} 448 | get_index(Rhc, Bucket, Index, Query) -> 449 | get_index(Rhc, Bucket, Index, Query, []). 450 | 451 | %% @doc Query a secondary index. 452 | %% @spec get_index(rhc(), bucket(), index(), index_query(), index_options()) -> 453 | %% {ok, index_results()} | {error, term()} 454 | get_index(Rhc, Bucket, Index, Query, Options) -> 455 | {ok, ReqId} = stream_index(Rhc, Bucket, Index, Query, Options), 456 | rhc_index:wait_for_index(ReqId). 457 | 458 | %% @doc Query a secondary index, streaming the results back. 459 | %% @spec stream_index(rhc(), bucket(), index(), index_query()) -> 460 | %% {ok, reference()} | {error, term()} 461 | stream_index(Rhc, Bucket, Index, Query) -> 462 | stream_index(Rhc, Bucket, Index, Query, []). 463 | 464 | %% @doc Query a secondary index, streaming the results back. 465 | %% @spec stream_index(rhc(), bucket(), index(), index_query(), index_options()) -> 466 | %% {ok, reference()} | {error, term()} 467 | stream_index(Rhc, Bucket, Index, Query, Options) -> 468 | ParamList = rhc_index:query_options([{stream, true}|Options]), 469 | Url = index_url(Rhc, Bucket, Index, Query, ParamList), 470 | StartRef = make_ref(), 471 | Pid = spawn(rhc_index, index_acceptor, [self(), StartRef]), 472 | case request_stream(Pid, get, Url, [], [], Rhc) of 473 | {ok, ReqId} -> 474 | Pid ! {ibrowse_req_id, StartRef, ReqId}, 475 | {ok, StartRef}; 476 | {error, Error} -> 477 | {error, Error} 478 | end. 479 | 480 | %% @doc Get the properties of the given bucket. 481 | %% @spec get_bucket(rhc(), bucket()) -> {ok, proplist()}|{error, term()} 482 | get_bucket(Rhc, Bucket) -> 483 | Url = make_url(Rhc, Bucket, undefined, [{?Q_PROPS, ?Q_TRUE}, 484 | {?Q_KEYS, ?Q_FALSE}]), 485 | case request(get, Url, ["200"], [], [], Rhc) of 486 | {ok, "200", _Headers, Body} -> 487 | {struct, Response} = mochijson2:decode(Body), 488 | {struct, Props} = proplists:get_value(?JSON_PROPS, Response), 489 | {ok, rhc_bucket:erlify_props(Props)}; 490 | {error, Error} -> 491 | {error, Error} 492 | end. 493 | 494 | %% @doc Set the properties of the given bucket. 495 | %% 496 | %% Allowed properties are: 497 | %%
498 | %%
`n_val'
499 | %%
The 'N' value to use for storing data in this bucket
500 | %%
`allow_mult'
501 | %%
Whether or not this bucket should allow siblings to 502 | %% be created for its keys
503 | %%
504 | %% @spec set_bucket(rhc(), bucket(), proplist()) -> ok|{error, term()} 505 | set_bucket(Rhc, Bucket, Props0) -> 506 | Url = make_url(Rhc, Bucket, undefined, [{?Q_PROPS, ?Q_TRUE}]), 507 | Headers = [{"Content-Type", "application/json"}], 508 | Props = rhc_bucket:httpify_props(Props0), 509 | Body = mochijson2:encode({struct, [{?Q_PROPS, {struct, Props}}]}), 510 | case request(put, Url, ["204"], Headers, Body, Rhc) of 511 | {ok, "204", _Headers, _Body} -> ok; 512 | {error, Error} -> {error, Error} 513 | end. 514 | 515 | reset_bucket(Rhc, Bucket) -> 516 | Url = make_url(Rhc, Bucket, undefined, [{?Q_PROPS, ?Q_TRUE}]), 517 | case request(delete, Url, ["204"], [], [], Rhc) of 518 | {ok, "204", _Headers, _Body} -> ok; 519 | {error, Error} -> {error, Error} 520 | end. 521 | 522 | 523 | %% @doc Get the properties of the given bucket. 524 | %% @spec get_bucket_type (rhc(), bucket()) -> {ok, proplist()}|{error, term()} 525 | get_bucket_type(Rhc, Type) -> 526 | Url = make_url(Rhc, {Type, undefined}, undefined, [{?Q_PROPS, ?Q_TRUE}, 527 | {?Q_KEYS, ?Q_FALSE}]), 528 | case request(get, Url, ["200"], [], [], Rhc) of 529 | {ok, "200", _Headers, Body} -> 530 | {struct, Response} = mochijson2:decode(Body), 531 | {struct, Props} = proplists:get_value(?JSON_PROPS, Response), 532 | {ok, rhc_bucket:erlify_props(Props)}; 533 | {error, Error} -> 534 | {error, Error} 535 | end. 536 | 537 | %% @doc Set the properties of the given bucket type. 538 | %% 539 | %% @spec set_bucket_type(rhc(), bucket(), proplist()) -> ok|{error, term()} 540 | set_bucket_type(Rhc, Type, Props0) -> 541 | Url = make_url(Rhc, {Type, undefined}, undefined, [{?Q_PROPS, ?Q_TRUE}]), 542 | Headers = [{"Content-Type", "application/json"}], 543 | Props = rhc_bucket:httpify_props(Props0), 544 | Body = mochijson2:encode({struct, [{?Q_PROPS, {struct, Props}}]}), 545 | case request(put, Url, ["204"], Headers, Body, Rhc) of 546 | {ok, "204", _Headers, _Body} -> ok; 547 | {error, Error} -> {error, Error} 548 | end. 549 | 550 | %% @equiv mapred(Rhc, Inputs, Query, DEFAULT_TIMEOUT) 551 | mapred(Rhc, Inputs, Query) -> 552 | mapred(Rhc, Inputs, Query, ?DEFAULT_TIMEOUT). 553 | 554 | %% @doc Execute a map/reduce query. See {@link 555 | %% rhc_mapred:encode_mapred/2} for details of the allowed formats 556 | %% for `Inputs' and `Query'. 557 | %% @spec mapred(rhc(), rhc_mapred:map_input(), 558 | %% [rhc_mapred:query_part()], integer()) 559 | %% -> {ok, [rhc_mapred:phase_result()]}|{error, term()} 560 | mapred(Rhc, Inputs, Query, Timeout) -> 561 | {ok, ReqId} = mapred_stream(Rhc, Inputs, Query, self(), Timeout), 562 | rhc_mapred:wait_for_mapred(ReqId, Timeout). 563 | 564 | %% @equiv mapred_stream(Rhc, Inputs, Query, ClientPid, DEFAULT_TIMEOUT) 565 | mapred_stream(Rhc, Inputs, Query, ClientPid) -> 566 | mapred_stream(Rhc, Inputs, Query, ClientPid, ?DEFAULT_TIMEOUT). 567 | 568 | %% @doc Stream map/reduce results to a Pid. Messages sent to the Pid 569 | %% will be of the form `{reference(), message()}', 570 | %% where `message()' is one of: 571 | %%
572 | %%
`done'
573 | %%
query has completed, no more messages will be sent
574 | %%
`{mapred, integer(), mochijson()}'
575 | %%
partial results of a query the second item in the tuple 576 | %% is the (zero-indexed) phase number, and the third is the 577 | %% JSON-decoded results
578 | %%
`{error, term()}'
579 | %%
an error occurred
580 | %%
581 | %% @spec mapred_stream(rhc(), rhc_mapred:mapred_input(), 582 | %% [rhc_mapred:query_phase()], pid(), integer()) 583 | %% -> {ok, reference()}|{error, term()} 584 | mapred_stream(Rhc, Inputs, Query, ClientPid, Timeout) -> 585 | Url = mapred_url(Rhc), 586 | StartRef = make_ref(), 587 | Pid = spawn(rhc_mapred, mapred_acceptor, [ClientPid, StartRef, Timeout]), 588 | Headers = [{?HEAD_CTYPE, "application/json"}], 589 | Body = rhc_mapred:encode_mapred(Inputs, Query), 590 | case request_stream(Pid, post, Url, Headers, Body, Rhc) of 591 | {ok, ReqId} -> 592 | Pid ! {ibrowse_req_id, StartRef, ReqId}, 593 | {ok, StartRef}; 594 | {error, Error} -> {error, Error} 595 | end. 596 | 597 | %% @doc Execute a search query. This command will return an error 598 | %% unless executed against a Riak Search cluster. 599 | %% @spec search(rhc(), bucket(), string()) -> 600 | %% {ok, [rhc_mapred:phase_result()]}|{error, term()} 601 | search(Rhc, Bucket, SearchQuery) -> 602 | %% Run a Map/Reduce operation using reduce_identity to get a list 603 | %% of BKeys. 604 | IdentityQuery = [{reduce, {modfun, riak_kv_mapreduce, reduce_identity}, none, true}], 605 | case search(Rhc, Bucket, SearchQuery, IdentityQuery, ?DEFAULT_TIMEOUT) of 606 | {ok, [{_, Results}]} -> 607 | %% Unwrap the results. 608 | {ok, Results}; 609 | Other -> Other 610 | end. 611 | 612 | %% @doc Execute a search query and feed the results into a map/reduce 613 | %% query. See {@link rhc_mapred:encode_mapred/2} for details of 614 | %% the allowed formats for `MRQuery'. This command will return an error 615 | %% unless executed against a Riak Search cluster. 616 | %% @spec search(rhc(), bucket(), string(), 617 | %% [rhc_mapred:query_part()], integer()) -> 618 | %% {ok, [rhc_mapred:phase_result()]}|{error, term()} 619 | search(Rhc, Bucket, SearchQuery, MRQuery, Timeout) -> 620 | Inputs = {modfun, riak_search, mapred_search, [Bucket, SearchQuery]}, 621 | mapred(Rhc, Inputs, MRQuery, Timeout). 622 | 623 | %% @equiv mapred_bucket(Rhc, Bucket, Query, DEFAULT_TIMEOUT) 624 | mapred_bucket(Rhc, Bucket, Query) -> 625 | mapred_bucket(Rhc, Bucket, Query, ?DEFAULT_TIMEOUT). 626 | 627 | %% @doc Execute a map/reduce query over all keys in the given bucket. 628 | %% @spec mapred_bucket(rhc(), bucket(), [rhc_mapred:query_phase()], 629 | %% integer()) 630 | %% -> {ok, [rhc_mapred:phase_result()]}|{error, term()} 631 | mapred_bucket(Rhc, Bucket, Query, Timeout) -> 632 | {ok, ReqId} = mapred_bucket_stream(Rhc, Bucket, Query, self(), Timeout), 633 | rhc_mapred:wait_for_mapred(ReqId, Timeout). 634 | 635 | %% @doc Stream map/reduce results over all keys in a bucket to a Pid. 636 | %% Similar to {@link mapred_stream/5} 637 | %% @spec mapred_bucket_stream(rhc(), bucket(), 638 | %% [rhc_mapred:query_phase()], pid(), integer()) 639 | %% -> {ok, reference()}|{error, term()} 640 | mapred_bucket_stream(Rhc, Bucket, Query, ClientPid, Timeout) -> 641 | mapred_stream(Rhc, Bucket, Query, ClientPid, Timeout). 642 | 643 | %% @doc Fetches the representation of a convergent datatype from Riak. 644 | -spec fetch_type(rhc(), {BucketType::binary(), Bucket::binary()}, Key::binary()) -> 645 | {ok, riakc_datatype:datatype()} | {error, term()}. 646 | fetch_type(Rhc, BucketAndType, Key) -> 647 | fetch_type(Rhc, BucketAndType, Key, []). 648 | 649 | %% @doc Fetches the representation of a convergent datatype from Riak, 650 | %% using the given request options. 651 | -spec fetch_type(rhc(), {BucketType::binary(), Bucket::binary()}, Key::binary(), [proplists:property()]) -> 652 | {ok, riakc_datatype:datatype()} | {error, term()}. 653 | fetch_type(Rhc, BucketAndType, Key, Options) -> 654 | Query = fetch_type_q_params(Rhc, Options), 655 | Url = make_datatype_url(Rhc, BucketAndType, Key, Query), 656 | case request(get, Url, ["200"], [], [], Rhc) of 657 | {ok, _Status, _Headers, Body} -> 658 | {ok, rhc_dt:datatype_from_json(mochijson2:decode(Body))}; 659 | {error, Reason} -> 660 | {error, rhc_dt:decode_error(fetch, Reason)} 661 | end. 662 | 663 | %% @doc Updates the convergent datatype in Riak with local 664 | %% modifications stored in the container type. 665 | -spec update_type(rhc(), {BucketType::binary(), Bucket::binary()}, Key::binary(), 666 | Update::riakc_datatype:update(term())) -> 667 | ok | {ok, Key::binary()} | {ok, riakc_datatype:datatype()} | 668 | {ok, Key::binary(), riakc_datatype:datatype()} | {error, term()}. 669 | update_type(Rhc, BucketAndType, Key, Update) -> 670 | update_type(Rhc, BucketAndType, Key, Update, []). 671 | 672 | -spec update_type(rhc(), {BucketType::binary(), Bucket::binary()}, Key::binary(), 673 | Update::riakc_datatype:update(term()), [proplists:property()]) -> 674 | ok | {ok, Key::binary()} | {ok, riakc_datatype:datatype()} | 675 | {ok, Key::binary(), riakc_datatype:datatype()} | {error, term()}. 676 | update_type(_Rhc, _BucketAndType, _Key, undefined, _Options) -> 677 | {error, unmodified}; 678 | update_type(Rhc, BucketAndType, Key, {Type, Op, Context}, Options) -> 679 | Query = update_type_q_params(Rhc, Options), 680 | Url = make_datatype_url(Rhc, BucketAndType, Key, Query), 681 | Body = mochijson2:encode(rhc_dt:encode_update_request(Type, Op, Context)), 682 | case request(post, Url, ["200", "201", "204"], 683 | [{"Content-Type", "application/json"}], Body, Rhc) of 684 | {ok, "204", _H, _B} -> 685 | %% not creation, no returnbody 686 | ok; 687 | {ok, "200", _H, RBody} -> 688 | %% returnbody was specified 689 | {ok, rhc_dt:datatype_from_json(mochijson2:decode(RBody))}; 690 | {ok, "201", Headers, RBody} -> 691 | %% Riak-assigned key 692 | Url = proplists:get_value("Location", Headers), 693 | Key = unicode:characters_to_binary(lists:last(string:tokens(Url, "/")), utf8), 694 | case proplists:get_value("Content-Length", Headers) of 695 | "0" -> 696 | {ok, Key}; 697 | _ -> 698 | {ok, Key, rhc_dt:datatype_from_json(mochijson2:decode(RBody))} 699 | end; 700 | {error, Reason} -> 701 | {error, rhc_dt:decode_error(update, Reason)} 702 | end. 703 | 704 | %% @doc Fetches, applies the given function to the value, and then 705 | %% updates the datatype in Riak. If an existing value is not found, 706 | %% but you want the updates to apply anyway, use the 'create' option. 707 | -spec modify_type(rhc(), fun((riakc_datatype:datatype()) -> riakc_datatype:datatype()), 708 | {BucketType::binary(), Bucket::binary()}, Key::binary(), [proplists:property()]) -> 709 | ok | {ok, riakc_datatype:datatype()} | {error, term()}. 710 | modify_type(Rhc, Fun, BucketAndType, Key, Options) -> 711 | Create = proplists:get_value(create, Options, true), 712 | case fetch_type(Rhc, BucketAndType, Key, Options) of 713 | {ok, Data} -> 714 | NewData = Fun(Data), 715 | Mod = riakc_datatype:module_for_term(NewData), 716 | update_type(Rhc, BucketAndType, Key, Mod:to_op(NewData), Options); 717 | {error, {notfound, Type}} when Create -> 718 | %% Not found, but ok to create it 719 | Mod = riakc_datatype:module_for_type(Type), 720 | NewData = Fun(Mod:new()), 721 | update_type(Rhc, BucketAndType, Key, Mod:to_op(NewData), Options); 722 | {error, Reason} -> 723 | {error, Reason} 724 | end. 725 | 726 | %% @doc Get the active preflist based on a particular bucket/key 727 | %% combination. 728 | -spec get_preflist(rhc(), binary(), binary()) -> {ok, [tuple()]}|{error, term()}. 729 | get_preflist(Rhc, Bucket, Key) -> 730 | Url = make_preflist_url(Rhc, Bucket, Key), 731 | case request(get, Url, ["200"], [], [], Rhc) of 732 | {ok, "200", _Headers, Body} -> 733 | {struct, Response} = mochijson2:decode(Body), 734 | {ok, erlify_preflist(Response)}; 735 | {error, Error} -> 736 | {error, Error} 737 | end. 738 | 739 | 740 | %% INTERNAL 741 | 742 | %% @doc Get the client ID to use, given the passed options and client. 743 | %% Choose the client ID in Options before the one in the client. 744 | %% @spec client_id(rhc(), proplist()) -> client_id() 745 | client_id(#rhc{options=RhcOptions}, Options) -> 746 | case proplists:get_value(client_id, Options) of 747 | undefined -> 748 | proplists:get_value(client_id, RhcOptions); 749 | ClientId -> 750 | ClientId 751 | end. 752 | 753 | %% @doc Generate a random client ID. 754 | %% @spec random_client_id() -> client_id() 755 | random_client_id() -> 756 | {{Y,Mo,D},{H,Mi,S}} = erlang:universaltime(), 757 | {_,_,NowPart} = now(), 758 | Id = erlang:phash2([Y,Mo,D,H,Mi,S,node(),NowPart]), 759 | base64:encode_to_string(<>). 760 | 761 | %% @doc Assemble the root URL for the given client 762 | %% @spec root_url(rhc()) -> iolist() 763 | root_url(#rhc{ip=Ip, port=Port, options=Opts}) -> 764 | Proto = case proplists:get_value(is_ssl, Opts) of 765 | true -> 766 | "https"; 767 | _ -> 768 | "http" 769 | end, 770 | [Proto, "://",Ip,":",integer_to_list(Port),"/"]. 771 | 772 | %% @doc Assemble the URL for the map/reduce resource 773 | %% @spec mapred_url(rhc()) -> iolist() 774 | mapred_url(Rhc) -> 775 | binary_to_list(iolist_to_binary([root_url(Rhc), "mapred/?chunked=true"])). 776 | 777 | %% @doc Assemble the URL for the ping resource 778 | %% @spec ping_url(rhc()) -> iolist() 779 | ping_url(Rhc) -> 780 | binary_to_list(iolist_to_binary([root_url(Rhc), "ping/"])). 781 | 782 | %% @doc Assemble the URL for the stats resource 783 | %% @spec stats_url(rhc()) -> iolist() 784 | stats_url(Rhc) -> 785 | binary_to_list(iolist_to_binary([root_url(Rhc), "stats/"])). 786 | 787 | %% @doc Assemble the URL for the 2I resource 788 | index_url(Rhc, BucketAndType, Index, Query, Params) -> 789 | {Type, Bucket} = extract_bucket_type(BucketAndType), 790 | QuerySegments = index_query_segments(Query), 791 | IndexName = index_name(Index), 792 | lists:flatten( 793 | [root_url(Rhc), 794 | [ ["types", "/", mochiweb_util:quote_plus(Type), "/"] || Type =/= undefined ], 795 | "buckets", "/", mochiweb_util:quote_plus(Bucket), "/", "index", "/", IndexName, 796 | [ [ "/", QS] || QS <- QuerySegments ], 797 | [ ["?", mochiweb_util:urlencode(Params)] || Params =/= []]]). 798 | 799 | 800 | index_query_segments(B) when is_binary(B) -> 801 | [ B ]; 802 | index_query_segments(I) when is_integer(I) -> 803 | [ integer_to_list(I) ]; 804 | index_query_segments({B1, B2}) when is_binary(B1), 805 | is_binary(B2)-> 806 | [ B1, B2 ]; 807 | index_query_segments({I1, I2}) when is_integer(I1), 808 | is_integer(I2) -> 809 | [ integer_to_list(I1), integer_to_list(I2) ]; 810 | index_query_segments(_) -> []. 811 | 812 | index_name({binary_index, B}) -> 813 | [B, "_bin"]; 814 | index_name({integer_index, I}) -> 815 | [I, "_int"]; 816 | index_name(Idx) -> Idx. 817 | 818 | 819 | %% @doc Assemble the URL for the given bucket and key 820 | %% @spec make_url(rhc(), bucket(), key(), proplist()) -> iolist() 821 | make_url(Rhc=#rhc{}, BucketAndType, Key, Query) -> 822 | {Type, Bucket} = extract_bucket_type(BucketAndType), 823 | {IsKeys, IsProps, IsBuckets} = detect_bucket_flags(Query), 824 | lists:flatten( 825 | [root_url(Rhc), 826 | [ [ "types", "/", mochiweb_util:quote_plus(Type), "/"] || Type =/= undefined ], 827 | %% Prefix, "/", 828 | [ [ "buckets" ] || IsBuckets ], 829 | [ ["buckets", "/", mochiweb_util:quote_plus(Bucket),"/"] || Bucket =/= undefined ], 830 | [ [ "keys" ] || IsKeys ], 831 | [ [ "props" ] || IsProps ], 832 | [ ["keys", "/", mochiweb_util:quote_plus(Key), "/"] || 833 | Key =/= undefined andalso not IsKeys andalso not IsProps ], 834 | [ ["?", mochiweb_util:urlencode(Query)] || Query =/= [] ] 835 | ]). 836 | 837 | %% @doc Generate a preflist url. 838 | -spec make_preflist_url(rhc(), binary(), binary()) -> iolist(). 839 | make_preflist_url(Rhc, BucketAndType, Key) -> 840 | {Type, Bucket} = extract_bucket_type(BucketAndType), 841 | lists:flatten( 842 | [root_url(Rhc), 843 | [ [ "types", "/", Type, "/"] || Type =/= undefined ], 844 | [ ["buckets", "/", Bucket,"/"] || Bucket =/= undefined ], 845 | [ [ "keys", "/", Key,"/" ] || Key =/= undefined], 846 | [ ["preflist/"] ]]). 847 | 848 | %% @doc Generate a counter url. 849 | -spec make_counter_url(rhc(), term(), term(), list()) -> iolist(). 850 | make_counter_url(Rhc, Bucket, Key, Query) -> 851 | lists:flatten( 852 | [root_url(Rhc), 853 | <<"buckets">>, "/", mochiweb_util:quote_plus(Bucket), "/", 854 | <<"counters">>, "/", mochiweb_util:quote_plus(Key), "?", 855 | [ [mochiweb_util:urlencode(Query)] || Query =/= []]]). 856 | 857 | make_datatype_url(Rhc, BucketAndType, Key, Query) -> 858 | case extract_bucket_type(BucketAndType) of 859 | {undefined, _B} -> 860 | throw(default_bucket_type_disallowed); 861 | {Type, Bucket} -> 862 | lists:flatten( 863 | [root_url(Rhc), 864 | "types/", mochiweb_util:quote_plus(Type), 865 | "/buckets/", mochiweb_util:quote_plus(Bucket), 866 | "/datatypes/", [ mochiweb_util:quote_plus(Key) || Key /= undefined ], 867 | [ ["?", mochiweb_util:urlencode(Query)] || Query /= [] ]]) 868 | end. 869 | 870 | %% @doc send an ibrowse request 871 | request(Method, Url, Expect, Headers, Body, Rhc) -> 872 | AuthHeader = get_auth_header(Rhc#rhc.options), 873 | SSLOptions = get_ssl_options(Rhc#rhc.options), 874 | Accept = {"Accept", "multipart/mixed, */*;q=0.9"}, 875 | case ibrowse:send_req(Url, [Accept|Headers] ++ AuthHeader, Method, Body, 876 | [{response_format, binary}] ++ SSLOptions) of 877 | Resp={ok, Status, _, _} -> 878 | case lists:member(Status, Expect) of 879 | true -> Resp; 880 | false -> {error, Resp} 881 | end; 882 | Error -> 883 | Error 884 | end. 885 | 886 | %% @doc stream an ibrowse request 887 | request_stream(Pid, Method, Url, Headers, Body, Rhc) -> 888 | AuthHeader = get_auth_header(Rhc#rhc.options), 889 | SSLOptions = get_ssl_options(Rhc#rhc.options), 890 | case ibrowse:send_req(Url, Headers ++ AuthHeader, Method, Body, 891 | [{stream_to, Pid}, 892 | {response_format, binary}] ++ SSLOptions) of 893 | {ibrowse_req_id, ReqId} -> 894 | {ok, ReqId}; 895 | Error -> 896 | Error 897 | end. 898 | 899 | %% @doc Get the default options for the given client 900 | %% @spec options(rhc()) -> proplist() 901 | options(#rhc{options=Options}) -> 902 | Options. 903 | 904 | %% @doc Extract the list of query parameters to use for a GET 905 | %% @spec get_q_params(rhc(), proplist()) -> proplist() 906 | get_q_params(Rhc, Options) -> 907 | options_list([r,pr,timeout], Options ++ options(Rhc)). 908 | 909 | %% @doc Extract the list of query parameters to use for a PUT 910 | %% @spec put_q_params(rhc(), proplist()) -> proplist() 911 | put_q_params(Rhc, Options) -> 912 | options_list([r,w,dw,pr,pw,timeout,asis,{return_body,"returnbody"}], 913 | Options ++ options(Rhc)). 914 | 915 | %% @doc Extract the list of query parameters to use for a 916 | %% counter increment 917 | -spec counter_q_params(rhc(), list()) -> list(). 918 | counter_q_params(Rhc, Options) -> 919 | options_list([r, pr, w, pw, dw, returnvalue, basic_quorum, notfound_ok], Options ++ options(Rhc)). 920 | 921 | %% @doc Extract the list of query parameters to use for a DELETE 922 | %% @spec delete_q_params(rhc(), proplist()) -> proplist() 923 | delete_q_params(Rhc, Options) -> 924 | options_list([r,w,dw,pr,pw,rw,timeout], Options ++ options(Rhc)). 925 | 926 | fetch_type_q_params(Rhc, Options) -> 927 | options_list([r,pr,basic_quorum,notfound_ok,timeout,include_context], Options ++ options(Rhc)). 928 | 929 | update_type_q_params(Rhc, Options) -> 930 | options_list([r,w,dw,pr,pw,basic_quorum,notfound_ok,timeout,include_context,{return_body, "returnbody"}], 931 | Options ++ options(Rhc)). 932 | 933 | %% @doc Extract the options for the given `Keys' from the possible 934 | %% list of `Options'. 935 | %% @spec options_list([Key::atom()|{Key::atom(),Alias::string()}], 936 | %% proplist()) -> proplist() 937 | options_list(Keys, Options) -> 938 | options_list(Keys, Options, []). 939 | 940 | options_list([K|Rest], Options, Acc) -> 941 | {Key,Alias} = case K of 942 | {_, _} -> K; 943 | _ -> {K, K} 944 | end, 945 | NewAcc = case proplists:lookup(Key, Options) of 946 | {Key,V} -> [{Alias,V}|Acc]; 947 | none -> Acc 948 | end, 949 | options_list(Rest, Options, NewAcc); 950 | options_list([], _, Acc) -> 951 | Acc. 952 | 953 | %% @doc Convert a stats-resource response to an erlang-term server 954 | %% information proplist. 955 | erlify_server_info(Props) -> 956 | lists:flatten([ erlify_server_info(K, V) || {K, V} <- Props ]). 957 | erlify_server_info(<<"nodename">>, Name) -> {node, Name}; 958 | erlify_server_info(<<"riak_kv_version">>, Vsn) -> {server_version, Vsn}; 959 | erlify_server_info(_Ignore, _) -> []. 960 | 961 | %% @doc Convert a preflist resource response to a proplist. 962 | erlify_preflist(Response) -> 963 | Preflist = [V || {_, V} <- proplists:get_value(<<"preflist">>, Response)], 964 | [ lists:foldl(fun({K, V}, AccRec) -> erlify_preflist(K, V, AccRec) end, 965 | #preflist_item{}, P) || P <- Preflist ]. 966 | 967 | erlify_preflist(<<"partition">>, Partition, Acc) -> 968 | Acc#preflist_item{partition=Partition}; 969 | erlify_preflist(<<"node">>, Node, Acc) -> 970 | Acc#preflist_item{node=Node}; 971 | erlify_preflist(<<"primary">>, IfPrimary, Acc) -> 972 | Acc#preflist_item{primary=IfPrimary}. 973 | 974 | 975 | get_auth_header(Options) -> 976 | case lists:keyfind(credentials, 1, Options) of 977 | {credentials, User, Password} -> 978 | [{"Authorization", "Basic " ++ base64:encode_to_string(User ++ ":" 979 | ++ 980 | Password)}]; 981 | _ -> 982 | [] 983 | end. 984 | 985 | get_ssl_options(Options) -> 986 | case proplists:get_value(is_ssl, Options) of 987 | true -> 988 | [{is_ssl, true}] ++ case proplists:get_value(ssl_options, Options, []) of 989 | X when is_list(X) -> 990 | [{ssl_options, X}]; 991 | _ -> 992 | [{ssl_options, []}] 993 | end; 994 | _ -> 995 | [] 996 | end. 997 | 998 | extract_bucket_type({<<"default">>, B}) -> 999 | {undefined, B}; 1000 | extract_bucket_type({T,B}) -> 1001 | {T,B}; 1002 | extract_bucket_type(B) -> 1003 | {undefined, B}. 1004 | 1005 | detect_bucket_flags(Query) -> 1006 | {proplists:get_value(?Q_KEYS, Query, ?Q_FALSE) =/= ?Q_FALSE, 1007 | proplists:get_value(?Q_PROPS, Query, ?Q_FALSE) =/= ?Q_FALSE, 1008 | proplists:get_value(?Q_BUCKETS, Query, ?Q_FALSE) =/= ?Q_FALSE}. 1009 | 1010 | -ifdef(TEST). 1011 | %% @doc validate that bucket, keys and link specifications do not contain 1012 | %% unescaped slashes 1013 | %% 1014 | %% See section on URL Escaping information at 1015 | %% http://docs.basho.com/riak/latest/dev/references/http/ 1016 | 1017 | url_escaping_test() -> 1018 | Rhc = create(), 1019 | Type = "my/type", 1020 | Bucket = "my/bucket", 1021 | Key = "my/key", 1022 | Query = [], 1023 | 1024 | Url = iolist_to_binary(make_url(Rhc, {Type, Bucket}, Key, [])), 1025 | ExpectedUrl = 1026 | <<"http://127.0.0.1:8098/types/my%2Ftype/buckets/my%2Fbucket/keys/my%2Fkey/">>, 1027 | ?assertEqual(ExpectedUrl, Url), 1028 | 1029 | CounterUrl = iolist_to_binary(make_counter_url(Rhc, Bucket, Key, Query)), 1030 | ExpectedCounterUrl = <<"http://127.0.0.1:8098/buckets/my%2Fbucket/counters/my%2Fkey?">>, 1031 | ?assertEqual(ExpectedCounterUrl, CounterUrl), 1032 | 1033 | IndexUrl = iolist_to_binary( 1034 | index_url(Rhc, {Type, Bucket}, {binary_index, "mybinaryindex"}, Query, [])), 1035 | ExpectedIndexUrl = 1036 | <<"http://127.0.0.1:8098/types/my%2Ftype/buckets/my%2Fbucket/index/mybinaryindex_bin">>, 1037 | ?assertEqual(ExpectedIndexUrl, IndexUrl), 1038 | 1039 | ok. 1040 | 1041 | -endif. 1042 | -------------------------------------------------------------------------------- /src/rhc_bucket.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riakhttpc: Riak HTTP Client 4 | %% 5 | %% Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | %% @doc This module contains utilities that the rhc module uses to 24 | %% encode and decode bucket-property requests. 25 | -module(rhc_bucket). 26 | 27 | -export([erlify_props/1, 28 | httpify_props/1]). 29 | 30 | -include("raw_http.hrl"). 31 | 32 | erlify_props(Props) -> 33 | lists:flatten([ erlify_prop(K, V) || {K, V} <- Props ]). 34 | erlify_prop(?JSON_ALLOW_MULT, AM) -> {allow_mult, AM}; 35 | erlify_prop(?JSON_BACKEND, B) -> {backend, B}; 36 | erlify_prop(?JSON_BASIC_Q, B) -> {basic_quorum, B}; 37 | erlify_prop(?JSON_BIG_VC, I) -> {big_vclock, I}; 38 | erlify_prop(?JSON_CHASH, CH) -> {chash_keyfun, erlify_chash(CH)}; 39 | erlify_prop(?JSON_DW, DW) -> {dw, erlify_quorum(DW)}; 40 | erlify_prop(?JSON_LINKFUN, LF) -> {linkfun, erlify_linkfun(LF)}; 41 | erlify_prop(?JSON_LWW, LWW) -> {last_write_wins, LWW}; 42 | erlify_prop(?JSON_NF_OK, NF) -> {notfound_ok, NF}; 43 | erlify_prop(?JSON_N_VAL, N) -> {n_val, N}; 44 | erlify_prop(?JSON_OLD_VC, I) -> {old_vclock, I}; 45 | erlify_prop(?JSON_POSTCOMMIT, P) -> {postcommit, P}; 46 | erlify_prop(?JSON_PR, PR) -> {pr, erlify_quorum(PR)}; 47 | erlify_prop(?JSON_PRECOMMIT, P) -> {precommit, P}; 48 | erlify_prop(?JSON_PW, PW) -> {pw, erlify_quorum(PW)}; 49 | erlify_prop(?JSON_R, R) -> {r, erlify_quorum(R)}; 50 | erlify_prop(?JSON_REPL, R) -> {repl, erlify_repl(R)}; 51 | erlify_prop(?JSON_RW, RW) -> {rw, erlify_quorum(RW)}; 52 | erlify_prop(?JSON_SEARCH, B) -> {search, B}; 53 | erlify_prop(?JSON_SMALL_VC, I) -> {small_vclock, I}; 54 | erlify_prop(?JSON_W, W) -> {w, erlify_quorum(W)}; 55 | erlify_prop(?JSON_YOUNG_VC, I) -> {young_vclock, I}; 56 | erlify_prop(?JSON_HLL_PRECISION, P) -> {hll_precision, P}; 57 | erlify_prop(_Ignore, _) -> []. 58 | 59 | erlify_quorum(?JSON_ALL) -> all; 60 | erlify_quorum(?JSON_QUORUM) -> quorum; 61 | erlify_quorum(?JSON_ONE) -> one; 62 | erlify_quorum(I) when is_integer(I) -> I; 63 | erlify_quorum(_) -> undefined. 64 | 65 | erlify_repl(?JSON_REALTIME) -> realtime; 66 | erlify_repl(?JSON_FULLSYNC) -> fullsync; 67 | erlify_repl(?JSON_BOTH) -> true; %% both is equivalent to true, but only works in 1.2+ 68 | erlify_repl(true) -> true; 69 | erlify_repl(false) -> false; 70 | erlify_repl(_) -> undefined. 71 | 72 | erlify_chash({struct, [{?JSON_MOD, Mod}, {?JSON_FUN, Fun}]}=Struct) -> 73 | try 74 | {binary_to_existing_atom(Mod, utf8), binary_to_existing_atom(Fun, utf8)} 75 | catch 76 | error:badarg -> 77 | error_logger:warning_msg("Creating modfun atoms from JSON bucket property! ~p", [Struct]), 78 | {binary_to_atom(Mod, utf8), binary_to_atom(Fun, utf8)} 79 | end. 80 | 81 | erlify_linkfun(Struct) -> 82 | {Mod, Fun} = erlify_chash(Struct), 83 | {modfun, Mod, Fun}. 84 | 85 | httpify_props(Props) -> 86 | lists:flatten([ httpify_prop(K, V) || {K, V} <- Props ]). 87 | httpify_prop(allow_mult, AM) -> {?JSON_ALLOW_MULT, AM}; 88 | httpify_prop(backend, B) -> {?JSON_BACKEND, B}; 89 | httpify_prop(basic_quorum, B) -> {?JSON_BASIC_Q, B}; 90 | httpify_prop(big_vclock, VC) -> {?JSON_BIG_VC, VC}; 91 | httpify_prop(chash_keyfun, MF) -> {?JSON_CHASH, httpify_modfun(MF)}; 92 | httpify_prop(dw, Q) -> {?JSON_DW, Q}; 93 | httpify_prop(last_write_wins, LWW) -> {?JSON_LWW, LWW}; 94 | httpify_prop(linkfun, LF) -> {?JSON_LINKFUN, httpify_modfun(LF)}; 95 | httpify_prop(n_val, N) -> {?JSON_N_VAL, N}; 96 | httpify_prop(notfound_ok, NF) -> {?JSON_NF_OK, NF}; 97 | httpify_prop(old_vclock, VC) -> {?JSON_OLD_VC, VC}; 98 | httpify_prop(postcommit, P) -> {?JSON_POSTCOMMIT, P}; 99 | httpify_prop(pr, Q) -> {?JSON_PR, Q}; 100 | httpify_prop(precommit, P) -> {?JSON_PRECOMMIT, P}; 101 | httpify_prop(pw, Q) -> {?JSON_PW, Q}; 102 | httpify_prop(r, Q) -> {?JSON_R, Q}; 103 | httpify_prop(repl, R) -> {?JSON_REPL, R}; 104 | httpify_prop(rw, Q) -> {?JSON_RW, Q}; 105 | httpify_prop(search, B) -> {?JSON_SEARCH, B}; 106 | httpify_prop(small_vclock, VC) -> {?JSON_SMALL_VC, VC}; 107 | httpify_prop(w, Q) -> {?JSON_W, Q}; 108 | httpify_prop(young_vclock, VC) -> {?JSON_YOUNG_VC, VC}; 109 | httpify_prop(hll_precision, P) -> {?JSON_HLL_PRECISION, P}; 110 | httpify_prop(_Ignore, _) -> []. 111 | 112 | httpify_modfun({modfun, M, F}) -> 113 | httpify_modfun({M, F}); 114 | httpify_modfun({M, F}) -> 115 | {struct, [{?JSON_MOD, atom_to_binary(M, utf8)}, 116 | {?JSON_FUN, atom_to_binary(F, utf8)}]}. 117 | -------------------------------------------------------------------------------- /src/rhc_dt.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riakhttpc: Riak HTTP Client 4 | %% 5 | %% Copyright (c) 2013 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | %% @doc Utility functions for datatypes. 24 | -module(rhc_dt). 25 | 26 | -export([ 27 | datatype_from_json/1, 28 | encode_update_request/3, 29 | decode_error/2 30 | ]). 31 | 32 | -define(FIELD_PATTERN, "^(.*)_(counter|set|hll|register|flag|map)$"). 33 | 34 | datatype_from_json({struct, Props}) -> 35 | Value = proplists:get_value(<<"value">>, Props), 36 | Type = binary_to_existing_atom(proplists:get_value(<<"type">>, Props), utf8), 37 | Context = proplists:get_value(<<"context">>, Props, undefined), 38 | Mod = riakc_datatype:module_for_type(Type), 39 | Mod:new(decode_value(Type, Value), Context). 40 | 41 | decode_value(counter, Value) -> Value; 42 | decode_value(set, Value) -> Value; 43 | decode_value(gset, Value) -> Value; 44 | decode_value(hll, Value) -> Value; 45 | decode_value(flag, Value) -> Value; 46 | decode_value(register, Value) -> Value; 47 | decode_value(map, {struct, Fields}) -> 48 | [ begin 49 | {Name, Type} = field_from_json(Field), 50 | {{Name,Type}, decode_value(Type, Value)} 51 | end || {Field, Value} <- Fields ]. 52 | 53 | field_from_json(Bin) when is_binary(Bin) -> 54 | {match, [Name, BinType]} = re:run(Bin, ?FIELD_PATTERN, [anchored, {capture, all_but_first, binary}]), 55 | {Name, binary_to_existing_atom(BinType, utf8)}. 56 | 57 | field_to_json({Name, Type}) when is_binary(Name), is_atom(Type) -> 58 | BinType = atom_to_binary(Type, utf8), 59 | <>. 60 | 61 | decode_error(fetch, {ok, "404", Headers, Body}) -> 62 | case proplists:get_value("Content-Type", Headers) of 63 | "application/json" -> 64 | %% We need to extract the type when not found 65 | {struct, Props} = mochijson2:decode(Body), 66 | Type = binary_to_existing_atom(proplists:get_value(<<"type">>, Props), utf8), 67 | {notfound, Type}; 68 | "text/" ++ _ -> 69 | Body 70 | end; 71 | decode_error(_, {ok, "400", _, Body}) -> 72 | {bad_request, Body}; 73 | decode_error(_, {ok, "301", _, Body}) -> 74 | {legacy_counter, Body}; 75 | decode_error(_, {ok, "403", _, Body}) -> 76 | {forbidden, Body}; 77 | decode_error(_, {ok, _, _, Body}) -> 78 | Body. 79 | 80 | encode_update_request(register, {assign, Bin}, _Context) -> 81 | {struct, [{<<"assign">>, Bin}]}; 82 | encode_update_request(flag, Atom, _Context) -> 83 | atom_to_binary(Atom, utf8); 84 | encode_update_request(counter, Op, _Context) -> 85 | {struct, [Op]}; 86 | encode_update_request(set, {update, Ops}, Context) -> 87 | {struct, Ops ++ include_context(Context)}; 88 | encode_update_request(set, Op, Context) -> 89 | {struct, [Op|include_context(Context)]}; 90 | encode_update_request(hll, {update, Ops}, Context) -> 91 | {struct, Ops ++ include_context(Context)}; 92 | encode_update_request(hll, Op, Context) -> 93 | {struct, [Op|include_context(Context)]}; 94 | encode_update_request(map, {update, Ops}, Context) -> 95 | {struct, orddict:to_list(lists:foldl(fun encode_map_op/2, orddict:new(), Ops)) ++ 96 | include_context(Context)}; 97 | encode_update_request(gset, {update, Ops}, Context) -> 98 | {struct, Ops ++ include_context(Context)}; 99 | encode_update_request(gset, Op, Context) -> 100 | {struct, [Op|include_context(Context)]}. 101 | 102 | 103 | encode_map_op({add, Entry}, Ops) -> 104 | orddict:append(add, field_to_json(Entry), Ops); 105 | encode_map_op({remove, Entry}, Ops) -> 106 | orddict:append(remove, field_to_json(Entry), Ops); 107 | encode_map_op({update, {_Key,Type}=Field, Op}, Ops) -> 108 | EncOp = encode_update_request(Type, Op, undefined), 109 | Update = {field_to_json(Field), EncOp}, 110 | case orddict:find(update, Ops) of 111 | {ok, {struct, Updates}} -> 112 | orddict:store(update, {struct, [Update|Updates]}, Ops); 113 | error -> 114 | orddict:store(update, {struct, [Update]}, Ops) 115 | end. 116 | 117 | include_context(undefined) -> []; 118 | include_context(<<>>) -> []; 119 | include_context(Bin) -> [{<<"context">>, Bin}]. 120 | 121 | -------------------------------------------------------------------------------- /src/rhc_index.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riakhttpc: Riak HTTP Client 4 | %% 5 | %% Copyright (c) 2013 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | %% @doc This module contains utilities that the rhc module uses to 24 | %% encode and decode secondary index queries. 25 | -module(rhc_index). 26 | 27 | -include_lib("riakc/include/riakc.hrl"). 28 | -include("raw_http.hrl"). 29 | -export([ 30 | query_options/1, 31 | wait_for_index/1, 32 | index_acceptor/2 33 | ]). 34 | 35 | %% @doc Extracts query-string options for 2i requests from the given 36 | %% options. 37 | -spec query_options([{atom(), term()}]) -> [{string(), string()}]. 38 | query_options(Options) -> 39 | lists:flatmap(fun query_option/1, Options). 40 | 41 | query_option({timeout, N}) when is_integer(N) -> 42 | [{?Q_TIMEOUT, integer_to_list(N)}]; 43 | query_option({stream, B}) when is_boolean(B) -> 44 | [{?Q_STREAM, atom_to_list(B)}]; 45 | query_option({max_results, N}) when is_integer(N) -> 46 | [{?Q_MAXRESULTS, integer_to_list(N)}]; 47 | query_option({continuation, C}) when is_binary(C) -> 48 | [{?Q_CONTINUATION, binary_to_list(C)}]; 49 | query_option({continuation, C}) when is_list(C) -> 50 | [{?Q_CONTINUATION, C}]; 51 | query_option({term_regex, C}) when is_binary(C) -> 52 | [{?Q_TERM_REGEX, binary_to_list(C)}]; 53 | query_option({term_regex, C}) when is_list(C) -> 54 | [{?Q_TERM_REGEX, C}]; 55 | query_option({return_terms, B}) when is_boolean(B) -> 56 | [{?Q_RETURNTERMS, atom_to_list(B)}]; 57 | query_option({pagination_sort, B}) when is_boolean(B) -> 58 | [{?Q_PAGINATION_SORT, atom_to_list(B)}]; 59 | query_option(_) -> 60 | []. 61 | 62 | %% @doc Collects 2i query results on behalf of the caller. 63 | -spec wait_for_index(reference()) -> {ok, ?INDEX_RESULTS{}} | {error, term()}. 64 | wait_for_index(ReqId) -> 65 | wait_for_index(ReqId, []). 66 | 67 | wait_for_index(ReqId, Acc) -> 68 | receive 69 | {ReqId, {done, Continuation}} -> 70 | {ok, collect_results(Acc, Continuation)}; 71 | {ReqId, done} -> 72 | {ok, collect_results(Acc, undefined)}; 73 | {ReqId, {error, Reason}} -> 74 | {error, Reason}; 75 | {ReqId, ?INDEX_STREAM_RESULT{}=Res} -> 76 | wait_for_index(ReqId, [Res|Acc]) 77 | end. 78 | 79 | collect_results(Acc, Continuation) -> 80 | lists:foldl(fun merge_index_results/2, 81 | ?INDEX_RESULTS{keys=[], 82 | terms=[], 83 | continuation=Continuation}, Acc). 84 | 85 | merge_index_results(?INDEX_STREAM_RESULT{keys=KL}, 86 | ?INDEX_RESULTS{keys=K0}=Acc) when is_list(KL) -> 87 | Acc?INDEX_RESULTS{keys=KL++K0}; 88 | merge_index_results(?INDEX_STREAM_RESULT{terms=TL}, 89 | ?INDEX_RESULTS{terms=T0}=Acc) when is_list(TL) -> 90 | Acc?INDEX_RESULTS{terms=TL++T0}. 91 | 92 | %% @doc The entry point for the middleman process that converts 93 | %% ibrowse streaming messages into index query results. 94 | -spec index_acceptor(pid(), reference()) -> no_return(). 95 | index_acceptor(Pid, PidRef) -> 96 | receive 97 | {ibrowse_req_id, PidRef, IbrowseRef} -> 98 | index_acceptor(Pid, PidRef, IbrowseRef) 99 | end. 100 | 101 | index_acceptor(Pid, PidRef, IBRef) -> 102 | receive 103 | {ibrowse_async_headers, IBRef, Status, Headers} -> 104 | case Status of 105 | "503" -> 106 | Pid ! {PidRef, {error, timeout}}; 107 | "200" -> 108 | {"multipart/mixed", Args} = rhc_obj:ctype_from_headers(Headers), 109 | Boundary = proplists:get_value("boundary", Args), 110 | stream_parts_acceptor( 111 | Pid, PidRef, 112 | webmachine_multipart:stream_parts( 113 | {[], stream_parts_helper(Pid, PidRef, IBRef, true)}, Boundary)); 114 | _ -> 115 | Pid ! {PidRef, {error, {Status, Headers}}} 116 | end 117 | end. 118 | 119 | %% @doc Receives multipart chunks from webmachine_multipart and parses 120 | %% them into results that can be sent to Pid. 121 | %% @private 122 | stream_parts_acceptor(Pid, PidRef, done_parts) -> 123 | Pid ! {PidRef, done}; 124 | stream_parts_acceptor(Pid, PidRef, {{_Name, _Param, Part},Next}) -> 125 | {struct, Response} = mochijson2:decode(Part), 126 | Keys = proplists:get_value(<<"keys">>, Response), 127 | Results = proplists:get_value(<<"results">>, Response), 128 | Continuation = proplists:get_value(<<"continuation">>, Response), 129 | maybe_send_results(Pid, PidRef, Keys, Results), 130 | maybe_send_continuation(Pid, PidRef, Continuation), 131 | stream_parts_acceptor(Pid, PidRef, Next()). 132 | 133 | %% @doc Sends keys or terms to the Pid if they are present in the 134 | %% result, otherwise sends nothing. 135 | %% @private 136 | maybe_send_results(_Pid, _PidRef, undefined, undefined) -> ok; 137 | maybe_send_results(Pid, PidRef, Keys, Results) -> 138 | Pid ! {PidRef, ?INDEX_STREAM_RESULT{keys=Keys, 139 | terms=Results}}. 140 | 141 | %% @doc Sends the continuation to Pid if it is present in the result, 142 | %% otherwise sends nothing. 143 | %% @private 144 | maybe_send_continuation(_Pid, _PidRef, undefined) -> ok; 145 | maybe_send_continuation(Pid, PidRef, Continuation) -> 146 | Pid ! {PidRef, {done, Continuation}}. 147 | 148 | %% @doc "next" fun for the webmachine_multipart streamer - waits for 149 | %% an ibrowse message, and then returns it to the streamer for processing 150 | %% @private 151 | stream_parts_helper(Pid, PidRef, IbrowseRef, First) -> 152 | fun() -> 153 | receive 154 | {ibrowse_async_response_end, IbrowseRef} -> 155 | {<<>>,done}; 156 | {ibrowse_async_response, IbrowseRef, {error, Error}} -> 157 | Pid ! {PidRef, {error, Error}}, 158 | throw({error, {ibrowse, Error}}); 159 | {ibrowse_async_response, IbrowseRef, []} -> 160 | Fun = stream_parts_helper(Pid, PidRef, IbrowseRef, First), 161 | Fun(); 162 | {ibrowse_async_response, IbrowseRef, Data0} -> 163 | %% the streamer doesn't like the body to start with 164 | %% CRLF, so strip that off on the first chunk 165 | Data = if First -> 166 | case Data0 of 167 | <<"\n",D/binary>> -> D; 168 | <<"\r\n",D/binary>> -> D; 169 | _ -> Data0 170 | end; 171 | true -> 172 | Data0 173 | end, 174 | {Data, 175 | stream_parts_helper(Pid, PidRef, IbrowseRef, false)} 176 | end 177 | end. 178 | -------------------------------------------------------------------------------- /src/rhc_listkeys.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riakhttpc: Riak HTTP Client 4 | %% 5 | %% Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | %% @doc This module contains utilities that the rhc module uses to 24 | %% parse list_keys request results. 25 | -module(rhc_listkeys). 26 | 27 | -export([wait_for_list/2]). 28 | %% spawnable exports 29 | -export([list_acceptor/3]). 30 | 31 | -include("raw_http.hrl"). 32 | -include("rhc.hrl"). 33 | 34 | -record(parse_state, {buffer=[], %% unused characters in reverse order 35 | brace=0, %% depth of braces in current partial 36 | quote=false, %% inside a quoted region? 37 | escape=false %% last character was escape? 38 | }). 39 | 40 | %% @doc Collect all keylist results, and provide them as one list 41 | %% instead of streaming to a Pid. 42 | %% @spec wait_for_list(term(), integer()) -> 43 | %% {ok, [key()]}|{error, term()} 44 | wait_for_list(ReqId, Timeout) -> 45 | wait_for_list(ReqId,Timeout,[]). 46 | %% @private 47 | wait_for_list(ReqId, _Timeout0, Acc) -> 48 | receive 49 | {ReqId, done} -> 50 | {ok, lists:append(Acc)}; 51 | {ReqId, {error, Reason}} -> 52 | {error, Reason}; 53 | {ReqId, {_,Res}} -> 54 | wait_for_list(ReqId,_Timeout0,[Res|Acc]) 55 | end. 56 | 57 | %% @doc first stage of ibrowse response handling - just waits to be 58 | %% told what ibrowse request ID to expect 59 | list_acceptor(Pid, PidRef, Type) -> 60 | receive 61 | {ibrowse_req_id, PidRef, IbrowseRef} -> 62 | list_acceptor(Pid,PidRef,IbrowseRef,#parse_state{},Type) 63 | end. 64 | 65 | %% @doc main loop for ibrowse response handling - parses response and 66 | %% sends messaged to client Pid 67 | list_acceptor(Pid,PidRef,IbrowseRef,ParseState,Type) -> 68 | receive 69 | {ibrowse_async_response_end, IbrowseRef} -> 70 | case is_empty(ParseState) of 71 | true -> 72 | Pid ! {PidRef, done}; 73 | false -> 74 | Pid ! {PidRef, {error, 75 | {not_parseable, 76 | ParseState#parse_state.buffer}}} 77 | end; 78 | {ibrowse_async_response, IbrowseRef, {error,Error}} -> 79 | Pid ! {PidRef, {error, Error}}; 80 | {ibrowse_async_response, IbrowseRef, []} -> 81 | %% ignore empty data 82 | ibrowse:stream_next(IbrowseRef), 83 | list_acceptor(Pid,PidRef,IbrowseRef,ParseState,Type); 84 | {ibrowse_async_response, IbrowseRef, Data} -> 85 | try 86 | {Keys, NewParseState} = try_parse(Data, ParseState, Type), 87 | if Keys =/= [] -> Pid ! {PidRef, {Type, Keys}}; 88 | true -> ok 89 | end, 90 | list_acceptor(Pid, PidRef, IbrowseRef, NewParseState,Type) 91 | catch 92 | Error -> 93 | Pid ! {PidRef, {error, Error}} 94 | end; 95 | {ibrowse_async_headers, IbrowseRef, Status, Headers} -> 96 | if Status =/= "200" -> 97 | Pid ! {PidRef, {error, {Status, Headers}}}; 98 | true -> 99 | ibrowse:stream_next(IbrowseRef), 100 | list_acceptor(Pid,PidRef,IbrowseRef,ParseState,Type) 101 | end 102 | end. 103 | 104 | is_empty(#parse_state{buffer=[],brace=0,quote=false,escape=false}) -> 105 | true; 106 | is_empty(#parse_state{}) -> 107 | false. 108 | 109 | try_parse(Data, #parse_state{buffer=B, brace=D, quote=Q, escape=E}, Type) -> 110 | Parse = try_parse(unicode:characters_to_list(Data, utf8), B, D, Q, E), 111 | {KeyLists, NewParseState} = 112 | lists:foldl( 113 | fun(Chunk, Acc) when is_list(Chunk), is_list(Acc) -> 114 | {struct, Props} = mochijson2:decode(Chunk), 115 | Keys = 116 | case proplists:get_value(<<"keys">>, Props, []) of 117 | [] -> 118 | proplists:get_value(<<"buckets">>, 119 | Props, []); 120 | K -> K 121 | end, 122 | case Keys of 123 | [] -> 124 | %%check for a timeout error 125 | case proplists:get_value(<<"error">>, Props, []) of 126 | [] -> Acc; 127 | Err -> throw(Err) 128 | end; 129 | _ -> [Keys|Acc] 130 | end; 131 | (PS=#parse_state{}, Acc) -> 132 | {Acc,PS} 133 | end, 134 | [], 135 | Parse), 136 | Keys = 137 | case Type of 138 | keys -> 139 | lists:flatten(KeyLists); 140 | buckets -> 141 | lists:flatten(KeyLists); 142 | ts_keys -> 143 | %% (a) avoid flattening TS keys, 144 | %% (b) present records as as tuples, as riakc does 145 | [list_to_tuple(X) || X <- lists:append(KeyLists)] 146 | end, 147 | {Keys, NewParseState}. 148 | 149 | try_parse([], B, D, Q, E) -> 150 | [#parse_state{buffer=B, brace=D, quote=Q, escape=E}]; 151 | try_parse([_|Rest],B,D,Q,true) -> 152 | try_parse(Rest,B,D,Q,false); 153 | try_parse([92|Rest],B,D,Q,false) -> %% backslash 154 | try_parse(Rest,B,D,Q,true); 155 | try_parse([34|Rest],B,D,Q,E) -> %% quote 156 | try_parse(Rest,[34|B],D,not Q,E); 157 | try_parse([123|Rest],B,D,Q,E) -> %% open brace 158 | if Q -> try_parse(Rest,[123|B],D,Q,E); 159 | true -> try_parse(Rest,[123|B],D+1,Q,E) 160 | end; 161 | try_parse([125|Rest],B,D,Q,E) -> %% close brace 162 | if Q -> try_parse(Rest,B,D,Q,E); 163 | true -> 164 | if D == 1 -> %% end of a chunk 165 | [lists:reverse([125|B]) 166 | |try_parse(Rest,[],0,Q,E)]; 167 | true -> 168 | try_parse(Rest,[125|B],D-1,Q,E) 169 | end 170 | end; 171 | try_parse([C|Rest],B,D,Q,E) -> 172 | try_parse(Rest,[C|B],D,Q,E). 173 | -------------------------------------------------------------------------------- /src/rhc_mapred.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riakhttpc: Riak HTTP Client 4 | %% 5 | %% Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | %% @doc This module contains utilities that the rhc module uses to 24 | %% encode and decode map/reduce queries. 25 | -module(rhc_mapred). 26 | 27 | -export([encode_mapred/2, 28 | wait_for_mapred/2]). 29 | %% spawnable exports 30 | -export([mapred_acceptor/3]). 31 | 32 | 33 | %%% REQUEST ENCODING 34 | 35 | %% @doc Translate erlang-term map/reduce query into JSON format. 36 | %% @spec encode_mapred(map_input(), [query_part()]) -> iolist() 37 | %% @type map_input() = bucket()|[key_spec()]| 38 | %% {modfun, atom(), atom(), term()} 39 | %% @type key_spec() = {bucket(), key()}|{{bucket(), key()},tag()} 40 | %% @type bucket() = binary() 41 | %% @type key() = binary() 42 | %% @type tag() = binary() 43 | %% @type query_part() = {map, funspec(), binary(), boolean()}| 44 | %% {reduce, funspec(), binary(), boolean()}| 45 | %% {link, linkspec(), linkspec(), boolean()} 46 | %% @type funspec() = {modfun, atom(), atom()}| 47 | %% {jsfun, binary()}| 48 | %% {jsanon, {bucket(), key()}}| 49 | %% {jsanon, binary()} 50 | %% @type linkspec() = binary()|'_' 51 | encode_mapred(Inputs, Query) -> 52 | mochijson2:encode( 53 | {struct, [{<<"inputs">>, encode_mapred_inputs(Inputs)}, 54 | {<<"query">>, encode_mapred_query(Query)}]}). 55 | encode_mapred_inputs({BucketType, Bucket}) when is_binary(BucketType), 56 | is_binary(Bucket) -> 57 | [BucketType, Bucket]; 58 | encode_mapred_inputs(Bucket) when is_binary(Bucket) -> 59 | Bucket; 60 | encode_mapred_inputs(Keylist) when is_list(Keylist) -> 61 | [ normalize_mapred_input(I) || I <- Keylist ]; 62 | encode_mapred_inputs({index, Bucket, Index, Key}) -> 63 | {struct, [{<<"bucket">>, encode_mapred_inputs(Bucket)}, 64 | {<<"index">>, riakc_obj:index_id_to_bin(Index)}, 65 | {<<"key">>, Key}]}; 66 | encode_mapred_inputs({index, Bucket, Index, StartKey, EndKey}) -> 67 | {struct, [{<<"bucket">>, encode_mapred_inputs(Bucket)}, 68 | {<<"index">>, riakc_obj:index_id_to_bin(Index)}, 69 | {<<"start">>, StartKey}, 70 | {<<"end">>, EndKey}]}; 71 | encode_mapred_inputs({modfun, Module, Function, Options}) -> 72 | {struct, [{<<"module">>, atom_to_binary(Module, utf8)}, 73 | {<<"function">>, atom_to_binary(Function, utf8)}, 74 | {<<"arg">>, Options}]}. 75 | 76 | %% @doc Normalize all bucket-key-data inputs to either 77 | %% [Bucket, Key] 78 | %% or 79 | %% [Bucket, Key, KeyData] 80 | normalize_mapred_input({Bucket, Key}) 81 | when is_binary(Bucket), is_binary(Key) -> 82 | [Bucket, Key]; 83 | normalize_mapred_input({{{Type, Bucket}, Key}, KeyData}) 84 | when is_binary(Type), is_binary(Bucket), is_binary(Key) -> 85 | [Bucket, Key, KeyData, Type]; 86 | normalize_mapred_input({{Bucket, Key}, KeyData}) 87 | when is_binary(Bucket), is_binary(Key) -> 88 | [Bucket, Key, KeyData]; 89 | normalize_mapred_input([Bucket, Key, _KeyData, Type]=List) 90 | when is_binary(Type), is_binary(Bucket), is_binary(Key) -> 91 | List; 92 | normalize_mapred_input([Bucket, Key]) 93 | when is_binary(Bucket), is_binary(Key) -> 94 | [Bucket, Key]; 95 | normalize_mapred_input([Bucket, Key, KeyData]) 96 | when is_binary(Bucket), is_binary(Key) -> 97 | [Bucket, Key, KeyData]. 98 | 99 | encode_mapred_query(Query) when is_list(Query) -> 100 | [ encode_mapred_phase(P) || P <- Query ]. 101 | 102 | encode_mapred_phase({MR, Fundef, Arg, Keep}) when MR =:= map; 103 | MR =:= reduce -> 104 | Type = if MR =:= map -> <<"map">>; 105 | MR =:= reduce -> <<"reduce">> 106 | end, 107 | {Lang, Json} = case Fundef of 108 | {modfun, Mod, Fun} -> 109 | {<<"erlang">>, 110 | [{<<"module">>, 111 | atom_to_binary(Mod, utf8)}, 112 | {<<"function">>, 113 | atom_to_binary(Fun, utf8)}]}; 114 | {jsfun, Name} -> 115 | {<<"javascript">>, 116 | [{<<"name">>, Name}]}; 117 | {jsanon, {Bucket, Key}} -> 118 | {<<"javascript">>, 119 | [{<<"bucket">>, Bucket}, 120 | {<<"key">>, Key}]}; 121 | {jsanon, Source} -> 122 | {<<"javascript">>, 123 | [{<<"source">>, Source}]} 124 | end, 125 | {struct, 126 | [{Type, 127 | {struct, [{<<"language">>, Lang}, 128 | {<<"arg">>, Arg}, 129 | {<<"keep">>, Keep} 130 | |Json 131 | ]} 132 | }]}; 133 | encode_mapred_phase({link, Bucket, Tag, Keep}) -> 134 | {struct, 135 | [{<<"link">>, 136 | {struct, [{<<"bucket">>, if Bucket =:= '_' -> <<"_">>; 137 | true -> Bucket 138 | end}, 139 | {<<"tag">>, if Tag =:= '_' -> <<"_">>; 140 | true -> Tag 141 | end}, 142 | {<<"keep">>, Keep} 143 | ]} 144 | }]}. 145 | 146 | %%% RESPONSE DECODING 147 | 148 | 149 | %% @doc Collect all mapreduce results, and provide them as one value 150 | %% instead of streaming to a Pid. 151 | %% @spec wait_for_mapred(term(), integer()) -> 152 | %% {ok, [phase_result()]}|{error, term()} 153 | %% @type phase_result() = {integer(), [term()]} 154 | wait_for_mapred(ReqId, Timeout) -> 155 | wait_for_mapred_first(ReqId, Timeout). 156 | 157 | %% Wait for the first mapred result, so we know at least one phase 158 | %% that will be delivering results. 159 | wait_for_mapred_first(ReqId, Timeout) -> 160 | case receive_mapred(ReqId, Timeout) of 161 | done -> 162 | {ok, []}; 163 | {mapred, Phase, Res} -> 164 | wait_for_mapred_one(ReqId, Timeout, Phase, 165 | acc_mapred_one(Res, [])); 166 | {error, _}=Error -> 167 | Error; 168 | timeout -> 169 | {error, {timeout, []}} 170 | end. 171 | 172 | %% So far we have only received results from one phase. This method 173 | %% of accumulating a single phases's outputs will be more efficient 174 | %% than the repeated orddict:append_list/3 used when accumulating 175 | %% outputs from multiple phases. 176 | wait_for_mapred_one(ReqId, Timeout, Phase, Acc) -> 177 | case receive_mapred(ReqId, Timeout) of 178 | done -> 179 | {ok, finish_mapred_one(Phase, Acc)}; 180 | {mapred, Phase, Res} -> 181 | %% still receiving for just one phase 182 | wait_for_mapred_one(ReqId, Timeout, Phase, 183 | acc_mapred_one(Res, Acc)); 184 | {mapred, NewPhase, Res} -> 185 | %% results from a new phase have arrived - track them all 186 | Dict = [{NewPhase, Res},{Phase, Acc}], 187 | wait_for_mapred_many(ReqId, Timeout, Dict); 188 | {error, _}=Error -> 189 | Error; 190 | timeout -> 191 | {error, {timeout, finish_mapred_one(Phase, Acc)}} 192 | end. 193 | 194 | %% Single-phase outputs are kept as a reverse list of results. 195 | acc_mapred_one([R|Rest], Acc) -> 196 | acc_mapred_one(Rest, [R|Acc]); 197 | acc_mapred_one([], Acc) -> 198 | Acc. 199 | 200 | finish_mapred_one(Phase, Acc) -> 201 | [{Phase, lists:reverse(Acc)}]. 202 | 203 | %% Tracking outputs from multiple phases. 204 | wait_for_mapred_many(ReqId, Timeout, Acc) -> 205 | case receive_mapred(ReqId, Timeout) of 206 | done -> 207 | {ok, finish_mapred_many(Acc)}; 208 | {mapred, Phase, Res} -> 209 | wait_for_mapred_many( 210 | ReqId, Timeout, acc_mapred_many(Phase, Res, Acc)); 211 | {error, _}=Error -> 212 | Error; 213 | timeout -> 214 | {error, {timeout, finish_mapred_many(Acc)}} 215 | end. 216 | 217 | %% Many-phase outputs are kepts as a proplist of reversed lists of 218 | %% results. 219 | acc_mapred_many(Phase, Res, Acc) -> 220 | case lists:keytake(Phase, 1, Acc) of 221 | {value, {Phase, PAcc}, RAcc} -> 222 | [{Phase,acc_mapred_one(Res,PAcc)}|RAcc]; 223 | false -> 224 | [{Phase,acc_mapred_one(Res,[])}|Acc] 225 | end. 226 | 227 | finish_mapred_many(Acc) -> 228 | [ {P, lists:reverse(A)} || {P, A} <- lists:keysort(1, Acc) ]. 229 | 230 | %% Receive one mapred message. 231 | -spec receive_mapred(reference(), timeout()) -> 232 | done | {mapred, integer(), [term()]} | {error, term()} | timeout. 233 | receive_mapred(ReqId, Timeout) -> 234 | receive {ReqId, Msg} -> 235 | %% Msg should be `done', `{mapred, Phase, Results}', or 236 | %% `{error, Reason}' 237 | Msg 238 | after Timeout -> 239 | timeout 240 | end. 241 | 242 | %% @doc first stage of ibrowse response handling - just waits to be 243 | %% told what ibrowse request ID to expect 244 | mapred_acceptor(Pid, PidRef, Timeout) -> 245 | receive 246 | {ibrowse_req_id, PidRef, IbrowseRef} -> 247 | mapred_acceptor(Pid,PidRef,Timeout,IbrowseRef) 248 | after Timeout -> 249 | Pid ! {PidRef, {error, {timeout, []}}} 250 | end. 251 | 252 | %% @doc second stage of ibrowse response handling - waits for headers 253 | %% and extracts the boundary of the multipart/mixed message 254 | mapred_acceptor(Pid,PidRef,Timeout,IbrowseRef) -> 255 | receive 256 | {ibrowse_async_headers, IbrowseRef, Status, Headers} -> 257 | if Status =/= "200" -> 258 | Pid ! {PidRef, {error, {Status, Headers}}}; 259 | true -> 260 | {"multipart/mixed", Args} = 261 | rhc_obj:ctype_from_headers(Headers), 262 | {"boundary", Boundary} = 263 | proplists:lookup("boundary", Args), 264 | stream_parts_acceptor( 265 | Pid, PidRef, 266 | webmachine_multipart:stream_parts( 267 | {[],stream_parts_helper(Pid,PidRef,Timeout, 268 | IbrowseRef,true)}, 269 | Boundary)) 270 | end 271 | after Timeout -> 272 | Pid ! {PidRef, {error, timeout}} 273 | end. 274 | 275 | %% @doc driver of the webmachine_multipart streamer - handles results 276 | %% of the parsing process (sends them to the client) and polls for 277 | %% the next part 278 | stream_parts_acceptor(Pid,PidRef,done_parts) -> 279 | Pid ! {PidRef, done}; 280 | stream_parts_acceptor(Pid,PidRef,{{_Name, _Param, Part},Next}) -> 281 | {struct, Response} = mochijson2:decode(Part), 282 | Phase = proplists:get_value(<<"phase">>, Response), 283 | Res = proplists:get_value(<<"data">>, Response), 284 | Pid ! {PidRef, {mapred, Phase, Res}}, 285 | stream_parts_acceptor(Pid,PidRef,Next()). 286 | 287 | %% @doc "next" fun for the webmachine_multipart streamer - waits for 288 | %% an ibrowse message, and then returns it to the streamer for processing 289 | stream_parts_helper(Pid, PidRef, Timeout, IbrowseRef, First) -> 290 | fun() -> 291 | receive 292 | {ibrowse_async_response_end, IbrowseRef} -> 293 | {<<>>,done}; 294 | {ibrowse_async_response, IbrowseRef, {error, Error}} -> 295 | Pid ! {PidRef, {error, Error}}, 296 | throw({error, {ibrowse, Error}}); 297 | {ibrowse_async_response, IbrowseRef, []} -> 298 | Fun = stream_parts_helper(Pid, PidRef, Timeout, 299 | IbrowseRef, First), 300 | Fun(); 301 | {ibrowse_async_response, IbrowseRef, Data0} -> 302 | %% the streamer doesn't like the body to start with 303 | %% CRLF, so strip that off on the first chunk 304 | Data = if First -> 305 | case Data0 of 306 | <<"\n",D/binary>> -> D; 307 | <<"\r\n",D/binary>> -> D; 308 | _ -> Data0 309 | end; 310 | true -> 311 | Data0 312 | end, 313 | {Data, 314 | stream_parts_helper(Pid, PidRef, Timeout, 315 | IbrowseRef, false)} 316 | after Timeout -> 317 | Pid ! {PidRef, {error, timeout}}, 318 | throw({error, {ibrowse, timeout}}) 319 | end 320 | end. 321 | -------------------------------------------------------------------------------- /src/rhc_obj.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riakhttpc: Riak HTTP Client 4 | %% 5 | %% Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | %% @doc This module contains utilities that the rhc module uses for 24 | %% translating between HTTP (ibrowse) data and riakc_obj objects. 25 | -module(rhc_obj). 26 | 27 | -export([make_riakc_obj/4, 28 | serialize_riakc_obj/2, 29 | ctype_from_headers/1]). 30 | 31 | -include("raw_http.hrl"). 32 | -include("rhc.hrl"). 33 | 34 | %% HTTP -> riakc_obj 35 | 36 | make_riakc_obj(Bucket, Key, Headers, Body) -> 37 | Vclock = base64:decode(proplists:get_value(?HEAD_VCLOCK, Headers, "")), 38 | case ctype_from_headers(Headers) of 39 | {"multipart/mixed", Args} -> 40 | {"boundary", Boundary} = proplists:lookup("boundary", Args), 41 | riakc_obj:new_obj( 42 | Bucket, Key, Vclock, 43 | decode_siblings(Boundary, Body)); 44 | {_CType, _} -> 45 | riakc_obj:new_obj( 46 | Bucket, Key, Vclock, 47 | [{headers_to_metadata(Headers), Body}]) 48 | end. 49 | 50 | ctype_from_headers(Headers) -> 51 | mochiweb_util:parse_header( 52 | proplists:get_value(?HEAD_CTYPE, Headers)). 53 | 54 | vtag_from_headers(Headers) -> 55 | %% non-sibling uses ETag, sibling uses Etag 56 | %% (note different capitalization on 't') 57 | case proplists:lookup("ETag", Headers) of 58 | {"ETag", ETag} -> ETag; 59 | none -> proplists:get_value("Etag", Headers) 60 | end. 61 | 62 | 63 | lastmod_from_headers(Headers) -> 64 | case proplists:get_value("Last-Modified", Headers) of 65 | undefined -> 66 | undefined; 67 | RfcDate -> 68 | GS = calendar:datetime_to_gregorian_seconds( 69 | httpd_util:convert_request_date(RfcDate)), 70 | ES = GS-62167219200, %% gregorian seconds of the epoch 71 | {ES div 1000000, % Megaseconds 72 | ES rem 1000000, % Seconds 73 | 0} % Microseconds 74 | end. 75 | 76 | decode_siblings(Boundary, <<"\r\n",SibBody/binary>>) -> 77 | decode_siblings(Boundary, SibBody); 78 | decode_siblings(Boundary, SibBody) -> 79 | Parts = webmachine_multipart:get_all_parts( 80 | SibBody, Boundary), 81 | [ {headers_to_metadata([ {binary_to_list(H), binary_to_list(V)} 82 | || {H, V} <- Headers ]), 83 | element(1, split_binary(Body, size(Body)))} 84 | || {_, {_, Headers}, Body} <- Parts ]. 85 | 86 | headers_to_metadata(Headers) -> 87 | UserMeta = extract_user_metadata(Headers), 88 | 89 | {CType,_} = ctype_from_headers(Headers), 90 | CUserMeta = dict:store(?MD_CTYPE, CType, UserMeta), 91 | 92 | VTag = vtag_from_headers(Headers), 93 | VCUserMeta = dict:store(?MD_VTAG, VTag, CUserMeta), 94 | 95 | LVCUserMeta = case lastmod_from_headers(Headers) of 96 | undefined -> 97 | VCUserMeta; 98 | LastMod -> 99 | dict:store(?MD_LASTMOD, LastMod, VCUserMeta) 100 | end, 101 | 102 | LinkMeta = case extract_links(Headers) of 103 | [] -> LVCUserMeta; 104 | Links -> dict:store(?MD_LINKS, Links, LVCUserMeta) 105 | end, 106 | case extract_indexes(Headers) of 107 | [] -> LinkMeta; 108 | Entries -> dict:store(?MD_INDEX, Entries, LinkMeta) 109 | end. 110 | 111 | extract_user_metadata(Headers) -> 112 | lists:foldl(fun extract_user_metadata/2, dict:new(), Headers). 113 | 114 | extract_user_metadata({?HEAD_USERMETA_PREFIX++K, V}, Dict) -> 115 | riakc_obj:set_user_metadata_entry(Dict, {K, V}); 116 | extract_user_metadata(_, D) -> D. 117 | 118 | extract_links(Headers) -> 119 | {ok, Re} = re:compile("; *riaktag=\"(.*)\""), 120 | Extractor = fun(L, Acc) -> 121 | case re:run(L, Re, [{capture,[1,2,3],binary}]) of 122 | {match, [Bucket, Key,Tag]} -> 123 | [{{Bucket,Key},Tag}|Acc]; 124 | nomatch -> 125 | Acc 126 | end 127 | end, 128 | LinkHeader = proplists:get_value(?HEAD_LINK, Headers, []), 129 | lists:foldl(Extractor, [], string:tokens(LinkHeader, ",")). 130 | 131 | extract_indexes(Headers) -> 132 | [ {list_to_binary(K), decode_index_value(K,V)} || {?HEAD_INDEX_PREFIX++K, V} <- Headers]. 133 | 134 | decode_index_value(K, V) -> 135 | case lists:last(string:tokens(K, "_")) of 136 | "bin" -> 137 | list_to_binary(V); 138 | "int" -> 139 | list_to_integer(V) 140 | end. 141 | 142 | %% riakc_obj -> HTTP 143 | 144 | serialize_riakc_obj(Rhc, Object) -> 145 | {make_headers(Rhc, Object), make_body(Object)}. 146 | 147 | make_headers(Rhc, Object) -> 148 | MD = riakc_obj:get_update_metadata(Object), 149 | CType = case dict:find(?MD_CTYPE, MD) of 150 | {ok, C} when is_list(C) -> C; 151 | {ok, C} when is_binary(C) -> binary_to_list(C); 152 | error -> "application/octet-stream" 153 | end, 154 | Links = case dict:find(?MD_LINKS, MD) of 155 | {ok, L} -> L; 156 | error -> [] 157 | end, 158 | VClock = riakc_obj:vclock(Object), 159 | lists:flatten( 160 | [{?HEAD_CTYPE, CType}, 161 | [ {?HEAD_LINK, encode_links(Rhc, Links)} || Links =/= [] ], 162 | [ {?HEAD_VCLOCK, base64:encode_to_string(VClock)} 163 | || VClock =/= undefined ], 164 | encode_indexes(MD) 165 | | encode_user_metadata(MD) ]). 166 | 167 | encode_links(_, []) -> []; 168 | encode_links(#rhc{prefix=Prefix}, Links) -> 169 | {{FirstBucket, FirstKey}, FirstTag} = hd(Links), 170 | lists:foldl( 171 | fun({{Bucket, Key}, Tag}, Acc) -> 172 | [format_link(Prefix, Bucket, Key, Tag), ", "|Acc] 173 | end, 174 | format_link(Prefix, FirstBucket, FirstKey, FirstTag), 175 | tl(Links)). 176 | 177 | encode_user_metadata(_Metadata) -> 178 | %% TODO 179 | []. 180 | 181 | encode_indexes(MD) -> 182 | case dict:find(?MD_INDEX, MD) of 183 | {ok, Entries} -> 184 | [ encode_index(Pair) || {_,_}=Pair <- Entries]; 185 | error -> 186 | [] 187 | end. 188 | 189 | encode_index({Name, IntValue}) when is_integer(IntValue) -> 190 | encode_index({Name, integer_to_list(IntValue)}); 191 | encode_index({Name, BinValue}) when is_binary(BinValue) -> 192 | encode_index({Name, unicode:characters_to_list(BinValue, latin1)}); 193 | encode_index({Name, String}) when is_list(String) -> 194 | {?HEAD_INDEX_PREFIX ++ unicode:characters_to_list(Name, latin1), 195 | String}. 196 | 197 | format_link(Prefix, Bucket, Key, Tag) -> 198 | io_lib:format("; riaktag=\"~s\"", 199 | [Prefix, Bucket, Key, Tag]). 200 | 201 | make_body(Object) -> 202 | case riakc_obj:get_update_value(Object) of 203 | Val when is_binary(Val) -> Val; 204 | Val when is_list(Val) -> 205 | case is_iolist(Val) of 206 | true -> Val; 207 | false -> term_to_binary(Val) 208 | end; 209 | Val -> 210 | term_to_binary(Val) 211 | end. 212 | 213 | is_iolist(Binary) when is_binary(Binary) -> true; 214 | is_iolist(List) when is_list(List) -> 215 | lists:all(fun is_iolist/1, List); 216 | is_iolist(_) -> false. 217 | -------------------------------------------------------------------------------- /src/rhc_ts.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% rhc_ts: TS extensions for Riak HTTP Client 4 | %% 5 | %% Copyright (c) 2016 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | %% @doc Riak TS Erlang HTTP Client. This module provides access to Riak's 24 | %% HTTP interface. For basic usage, please read 25 | %% the riakhttpc application overview. 26 | 27 | -module(rhc_ts). 28 | 29 | -export([create/0, create/3, 30 | ip/1, 31 | port/1, 32 | options/1, 33 | get_client_id/1, 34 | get/3, get/4, 35 | put/3, put/4, 36 | delete/3, delete/4, 37 | list_keys/2, 38 | stream_list_keys/2, 39 | 'query'/2 40 | ]). 41 | 42 | -include("raw_http.hrl"). 43 | -include("rhc.hrl"). 44 | -include_lib("riakc/include/riakc.hrl"). 45 | 46 | -type ts_table() :: binary() | string(). 47 | -type ts_key() :: [integer() | float() | binary()]. 48 | -type ts_record() :: [integer() | float() | binary()]. 49 | -type ts_selection() :: {Columns::[binary()], Rows::[ts_record()]}. 50 | 51 | -define(API_VERSION, "v1"). 52 | 53 | 54 | %% @doc Create a client for connecting to the default port on localhost. 55 | %% @equiv create("127.0.0.1", 8098, []). 56 | create() -> 57 | create("127.0.0.1", 8098, []). 58 | 59 | %% @doc Create a client for connecting to a Riak node. 60 | -spec create(string(), inet:port_number(), Options::list()) -> 61 | #rhc{}. 62 | %% @doc Create a client for connecting to IP:Port, with predefined 63 | %% request parameters and client options in Options, specially including: 64 | %% api_version :: string(), to insert after /ts/; 65 | %% client_id :: string(), to use in request headers; 66 | create(IP, Port, Options0) 67 | when is_list(IP), is_integer(Port), is_list(Options0) -> 68 | Options = 69 | lists:foldl( 70 | fun({Key, Default, ValidF}, AccOpts) -> 71 | lists:keystore( 72 | Key, 1, AccOpts, 73 | {Key, ValidF(proplists:get_value(Key, AccOpts, Default))}) 74 | end, 75 | Options0, 76 | [{api_version, ?API_VERSION, 77 | fun(X) when is_list(X) -> X end}, 78 | {client_id, random_client_id(), 79 | fun(X) when is_list(X) -> X end}]), 80 | #rhc{ip = IP, port = Port, 81 | prefix = "/ts/"++proplists:get_value(api_version, Options), 82 | options = Options}. 83 | 84 | %% @doc Get the IP this client will connect to. 85 | %% @spec ip(#rhc{}) -> string() 86 | ip(#rhc{ip=IP}) -> IP. 87 | 88 | %% @doc Get the Port this client will connect to. 89 | %% @spec port(#rhc{}) -> integer() 90 | port(#rhc{port=Port}) -> Port. 91 | 92 | %% @doc Get the client ID that this client will use when storing objects. 93 | %% @spec get_client_id(#rhc{}) -> {ok, string()} 94 | get_client_id(Rhc) -> 95 | {ok, client_id(Rhc, [])}. 96 | 97 | 98 | -spec get(#rhc{}, ts_table(), ts_key()) -> 99 | {ok, ts_selection()} | {error, term()}. 100 | %% @equiv get(Rhc, Table, Key, []) 101 | get(Rhc, Table, Key) -> 102 | get(Rhc, Table, Key, []). 103 | 104 | -spec get(#rhc{}, ts_table(), ts_key(), Options::proplists:proplist()) -> 105 | {ok, ts_selection()} | {error, notfound | {integer(), binary()} | term()}. 106 | %% @doc Get the TS record stored in Table at Key. 107 | %% Takes a value for timeout in Options. 108 | get(Rhc, Table, Key, Options) when is_binary(Table) -> 109 | get(Rhc, binary_to_list(Table), Key, Options); 110 | get(Rhc, Table, Key, Options) -> 111 | Qs = get_q_params(Rhc, Options), 112 | case make_get_url(Rhc, Table, Key, Qs, Options) of 113 | {ok, Url} -> 114 | Headers = [{?HEAD_CLIENT, client_id(Rhc, Options)}], 115 | case request(get, Url, ["200"], Headers, [], Rhc) of 116 | {ok, _Status, _Headers, Body} -> 117 | case catch mochijson2:decode(Body) of 118 | {struct, [{<<"columns">>, Columns}, {<<"rows">>, Rows}]} -> 119 | {ok, {Columns, Rows}}; 120 | _ -> 121 | {error, bad_body} 122 | end; 123 | {error, "404", _, _} -> 124 | {error, notfound}; 125 | {_, Code, _, Body} when is_list(Code) -> 126 | {error, {list_to_integer(Code), Body}}; 127 | {error, Error} -> 128 | {error, Error} 129 | end; 130 | {error, Reason} -> 131 | {error, Reason} 132 | end. 133 | 134 | -spec put(#rhc{}, ts_table(), [ts_record()]) -> ok | {error, {integer(), binary()} | term()}. 135 | put(Rhc, Table, Records) -> 136 | put(Rhc, Table, Records, []). 137 | 138 | -spec put(#rhc{}, ts_table(), Batch::[ts_record()], proplists:proplist()) -> 139 | ok | {error, {integer(), binary()} | term()}. 140 | %% @doc Batch put of TS records. 141 | put(Rhc, Table, Batch, Options) when is_binary(Table) -> 142 | put(Rhc, binary_to_list(Table), Batch, Options); 143 | put(Rhc, Table, Batch, Options) -> 144 | Encoded = mochijson2:encode(Batch), 145 | Qs = put_q_params(Rhc, Options), 146 | {ok, Url} = make_put_url(Rhc, Table, Qs), 147 | Headers = [{?HEAD_CLIENT, client_id(Rhc, Options)}], 148 | case request(post, Url, ["200", "401", "400"], Headers, Encoded, Rhc) of 149 | {ok, "200", _, <<"ok">>} -> 150 | ok; 151 | {ok, "200", _, <<"{\"success\":true}">>} -> 152 | ok; 153 | {ok, "404", _, _} -> 154 | {error, notfound}; 155 | {_, Code, _, Body} when is_list(Code) -> 156 | {error, {list_to_integer(Code), Body}}; 157 | {error, Error} -> 158 | {error, Error} 159 | end. 160 | 161 | 162 | -spec delete(#rhc{}, ts_table(), ts_key()) -> ok | {error, {integer(), binary()} | term()}. 163 | %% @equiv delete(Rhc, Table, Key, []) 164 | delete(Rhc, Table, Key) -> 165 | delete(Rhc, Table, Key, []). 166 | 167 | -spec delete(#rhc{}, ts_table(), ts_key(), proplists:proplist()) -> 168 | ok | {error, notfound | bad_key | {integer(), binary()} | term()}. 169 | %% @doc Delete the given key from the given bucket. 170 | delete(Rhc, Table, Key, Options) when is_binary(Table) -> 171 | delete(Rhc, binary_to_list(Table), Key, Options); 172 | delete(Rhc, Table, Key, Options) -> 173 | Qs = delete_q_params(Rhc, Options), 174 | case make_delete_url(Rhc, Table, Key, Qs, Options) of 175 | {ok, Url} -> 176 | Headers = [{?HEAD_CLIENT, client_id(Rhc, Options)}], 177 | case request(delete, Url, ["200", "401", "400"], Headers, [], Rhc) of 178 | {ok, "200", _Headers, <<"ok">>} -> 179 | ok; 180 | {ok, _Status, _, <<"{\"success\":true}">>} -> 181 | ok; 182 | {ok, "404", _, _} -> 183 | {error, notfound}; 184 | {ok, "400", _, _} -> 185 | {error, bad_key}; 186 | {_, Code, _, Body} when is_list(Code) -> 187 | {error, {list_to_integer(Code), Body}}; 188 | {error, Error} -> 189 | {error, Error} 190 | end; 191 | {error, Reason} -> 192 | {error, Reason} 193 | end. 194 | 195 | 196 | -spec list_keys(#rhc{}, ts_table()) -> {ok, [ts_key()]} | {error, notfound}. 197 | list_keys(Rhc, Table) -> 198 | {ok, ReqId} = stream_list_keys(Rhc, Table), 199 | case rhc_listkeys:wait_for_list(ReqId, ?DEFAULT_TIMEOUT) of 200 | {ok, {"404", _Headers}} -> 201 | {error, notfound}; 202 | Result -> 203 | Result 204 | end. 205 | 206 | -spec stream_list_keys(#rhc{}, ts_table()) -> 207 | {ok, reference()} | {error, term()}. 208 | %% @doc Stream key lists to a Pid. 209 | stream_list_keys(Rhc, Table) when is_binary(Table) -> 210 | stream_list_keys(Rhc, binary_to_list(Table)); 211 | stream_list_keys(Rhc, Table) -> 212 | Url = lists:flatten([root_url(Rhc),"/tables/",Table,"/list_keys"]), 213 | StartRef = make_ref(), 214 | Pid = spawn(rhc_listkeys, list_acceptor, [self(), StartRef, ts_keys]), 215 | case request_stream(Pid, get, Url, [], [], Rhc) of 216 | {ok, ReqId} -> 217 | Pid ! {ibrowse_req_id, StartRef, ReqId}, 218 | {ok, StartRef}; 219 | {error, Error} -> 220 | {error, Error} 221 | end. 222 | 223 | 224 | -spec 'query'(#rhc{}, string()) -> 225 | {ok, ts_selection()} | {error, bad_body | {integer(), binary()} | term()}. 226 | 'query'(Rhc, Query) -> 227 | 'query'(Rhc, Query, []). 228 | 229 | -spec 'query'(#rhc{}, string(), proplists:proplist()) -> 230 | ts_selection() | {error, bad_body | {integer(), binary()} | term()}. 231 | 'query'(Rhc, Query, Options) -> 232 | Url = lists:flatten([root_url(Rhc), "/query"]), 233 | Headers = [{?HEAD_CLIENT, client_id(Rhc, Options)}], 234 | case request(post, Url, ["200", "204", "401", "409", "400"], Headers, Query, Rhc) of 235 | {ok, TwoHundred, _Headers, BodyJson} 236 | when TwoHundred == "200"; 237 | TwoHundred == "204" -> 238 | case catch mochijson2:decode(BodyJson) of 239 | {struct, [{<<"success">>, true}]} -> 240 | %% idiosyncratic way to report {ok, {[], []}} 241 | {ok, {[], []}}; 242 | {struct, [{<<"columns">>, Columns}, {<<"rows">>, Rows}]} -> 243 | %% convert records (coming in as lists) to tuples 244 | %% to conform to the representation used in riakc 245 | {ok, {Columns, [list_to_tuple(R) || R <- Rows]}}; 246 | _Wat -> 247 | {error, bad_body} 248 | end; 249 | {_, Code, _, Body} when is_list(Code) -> 250 | {error, {list_to_integer(Code), Body}}; 251 | {error, Error} -> 252 | {error, Error} 253 | end. 254 | 255 | 256 | 257 | %% ------------------------ 258 | %% supporting functions 259 | 260 | %% @doc Get the client ID to use, given the passed options and client. 261 | %% Choose the client ID in Options before the one in the client. 262 | %% @spec client_id(#rhc{}, proplist()) -> client_id() 263 | client_id(#rhc{options = RhcOptions}, Options) -> 264 | proplists:get_value( 265 | client_id, Options, 266 | proplists:get_value(client_id, RhcOptions)). 267 | 268 | %% @doc Generate a random client ID. 269 | %% @spec random_client_id() -> client_id() 270 | random_client_id() -> 271 | Id = crypto:rand_bytes(32), 272 | integer_to_list(erlang:phash2(Id)). 273 | 274 | %% @doc Assemble the root URL for the given client 275 | %% @spec root_url(#rhc{}) -> iolist() 276 | root_url(#rhc{ip = Ip, port = Port, 277 | prefix = Prefix, options = Opts}) -> 278 | Proto = case proplists:get_value(is_ssl, Opts) of 279 | true -> 280 | "https"; 281 | _ -> 282 | "http" 283 | end, 284 | [Proto, "://",Ip,$:,integer_to_list(Port),$/,Prefix]. 285 | 286 | 287 | %% @doc send an ibrowse request 288 | request(Method, Url, Expect, Headers, Body, Rhc) -> 289 | AuthHeader = get_auth_header(Rhc#rhc.options), 290 | SSLOptions = get_ssl_options(Rhc#rhc.options), 291 | Accept = {"Accept", "multipart/mixed, */*;q=0.9"}, 292 | case ibrowse:send_req(Url, [Accept|Headers] ++ AuthHeader, Method, Body, 293 | [{response_format, binary}] ++ SSLOptions) of 294 | Resp = {ok, Status, _, _} -> 295 | case lists:member(Status, Expect) of 296 | true -> Resp; 297 | false -> {error, Resp} 298 | end; 299 | Error -> 300 | Error 301 | end. 302 | 303 | %% @doc stream an ibrowse request 304 | request_stream(Pid, Method, Url, Headers, Body, Rhc) -> 305 | AuthHeader = get_auth_header(Rhc#rhc.options), 306 | SSLOptions = get_ssl_options(Rhc#rhc.options), 307 | Accept = {"Accept", "multipart/mixed, */*;q=0.9"}, 308 | case ibrowse:send_req(Url, [Accept | Headers] ++ AuthHeader, Method, Body, 309 | [{stream_to, Pid}, 310 | {response_format, binary}] ++ SSLOptions) of 311 | {ibrowse_req_id, ReqId} -> 312 | {ok, ReqId}; 313 | Error -> 314 | Error 315 | end. 316 | 317 | %% @doc Get the default options for the given client 318 | %% @spec options(#rhc{}) -> proplist() 319 | options(#rhc{options=Options}) -> 320 | Options. 321 | 322 | %% @doc Extract the list of query parameters to use for a GET 323 | %% @spec get_q_params(#rhc{}, proplist()) -> proplist() 324 | get_q_params(Rhc, Options) -> 325 | options_list([timeout], 326 | Options ++ options(Rhc)). 327 | 328 | %% @doc Extract the list of query parameters to use for a PUT 329 | %% @spec put_q_params(#rhc{}, proplist()) -> proplist() 330 | put_q_params(Rhc, Options) -> 331 | options_list([], 332 | Options ++ options(Rhc)). 333 | 334 | %% @doc Extract the list of query parameters to use for a DELETE 335 | %% @spec delete_q_params(#rhc{}, proplist()) -> proplist() 336 | delete_q_params(Rhc, Options) -> 337 | options_list([timeout], 338 | Options ++ options(Rhc)). 339 | 340 | 341 | -spec options_list([Key::atom()|{Key::atom(),Alias::string()}], 342 | proplists:proplist()) -> proplists:proplist(). 343 | %% @doc Extract the options for the given `Keys' from the possible 344 | %% list of `Options'. 345 | options_list(Keys, Options) -> 346 | options_list(Keys, Options, []). 347 | 348 | options_list([K|Rest], Options, Acc) -> 349 | {Key,Alias} = case K of 350 | {_, _} -> K; 351 | _ -> {K, K} 352 | end, 353 | NewAcc = case proplists:lookup(Key, Options) of 354 | {Key,V} -> [{Alias,V}|Acc]; 355 | none -> Acc 356 | end, 357 | options_list(Rest, Options, NewAcc); 358 | options_list([], _, Acc) -> 359 | Acc. 360 | 361 | get_auth_header(Options) -> 362 | case lists:keyfind(credentials, 1, Options) of 363 | {credentials, User, Password} -> 364 | [{"Authorization", 365 | "Basic " ++ base64:encode_to_string(User ++ ":" ++ Password)}]; 366 | _ -> 367 | [] 368 | end. 369 | 370 | get_ssl_options(Options) -> 371 | case proplists:get_value(is_ssl, Options) of 372 | true -> 373 | [{is_ssl, true}] ++ case proplists:get_value(ssl_options, Options, []) of 374 | X when is_list(X) -> 375 | [{ssl_options, X}]; 376 | _ -> 377 | [{ssl_options, []}] 378 | end; 379 | _ -> 380 | [] 381 | end. 382 | 383 | make_get_url(Rhc, Table, Key, Qs, Options) -> 384 | make_keys_url(Rhc, Table, Key, Qs, Options). 385 | make_delete_url(Rhc, Table, Key, Qs, Options) -> 386 | make_keys_url(Rhc, Table, Key, Qs, Options). 387 | make_keys_url(Rhc, Table, Key, Qs, Options) -> 388 | %% serialize key elements as /f1/v1/f2/v2: 389 | %% 1. fetch field names, supplied separately in this call's Options 390 | case proplists:get_value(key_column_names, Options) of 391 | ColNames when is_list(ColNames), 392 | length(ColNames) == length(Key) -> 393 | EnrichedKey = lists:zip(ColNames, Key), 394 | {ok, lists:flatten( 395 | [root_url(Rhc), 396 | "/tables/", Table, 397 | keys_to_path_elements_append(EnrichedKey, "/keys"), 398 | [[$?, mochiweb_util:urlencode(Qs)] || Qs /= []]])}; 399 | undefined -> 400 | %% caller didn't specify column names: try v1/v2/v3 401 | {ok, lists:flatten( 402 | [root_url(Rhc), 403 | "/tables/", Table, 404 | keys_to_path_elements_append(Key, "/keys"), 405 | [[$?, mochiweb_util:urlencode(Qs)] || Qs /= []]])}; 406 | _ColNames when is_list(_ColNames) -> 407 | {error, key_col_count_mismatch}; 408 | _Invalid -> 409 | {error, invalid_col_names} 410 | end. 411 | 412 | keys_to_path_elements_append([], Acc) -> 413 | lists:flatten(Acc); 414 | keys_to_path_elements_append([{F, V} | T], Acc) -> 415 | keys_to_path_elements_append( 416 | T, lists:append( 417 | Acc, [$/, maybe_composite_field_to_list(F), 418 | $/, mochiweb_util:quote_plus(value_to_list(V))])); 419 | keys_to_path_elements_append([V | T], Acc) -> 420 | keys_to_path_elements_append( 421 | T, lists:append( 422 | Acc, [ %% bare values 423 | $/, mochiweb_util:quote_plus(value_to_list(V))])). 424 | 425 | maybe_composite_field_to_list(X) when is_binary(X) -> 426 | %% the user gave us a simplified representation of [<<"a">>] 427 | binary_to_list(X); 428 | maybe_composite_field_to_list(C = [E|_]) when is_binary(E) -> 429 | %% full representation 430 | string:join( 431 | [binary_to_list(X) || X <- C], "."). 432 | 433 | 434 | value_to_list(X) when is_binary(X) -> 435 | binary_to_list(X); 436 | value_to_list(X) when is_integer(X) -> 437 | integer_to_list(X); 438 | value_to_list(X) when is_float(X) -> 439 | float_to_list(X). 440 | 441 | make_put_url(Rhc, Table, Qs_) -> 442 | Qs = [{K, iolist_to_binary(V)} || {K,V} <- Qs_], 443 | {ok, lists:flatten( 444 | [root_url(Rhc), 445 | "/tables/", Table, "/keys/", 446 | [[$?, mochiweb_util:urlencode(Qs)] || Qs /= []]])}. 447 | -------------------------------------------------------------------------------- /src/riakhttpc.app.src: -------------------------------------------------------------------------------- 1 | %% -*- mode: Erlang -*- 2 | 3 | {application, riakhttpc, 4 | [ 5 | {description, "Riak HTTP Client"}, 6 | {vsn, git}, 7 | {modules, [ 8 | rhc, 9 | rhc_ts, 10 | rhc_bucket, 11 | rhc_dt, 12 | rhc_listkeys, 13 | rhc_index, 14 | rhc_mapred, 15 | rhc_obj 16 | ]}, 17 | {registered, []}, 18 | {applications, [ 19 | kernel, 20 | stdlib, 21 | crypto, 22 | ibrowse 23 | ]}, 24 | {env, []} 25 | ]}. 26 | -------------------------------------------------------------------------------- /tools.mk: -------------------------------------------------------------------------------- 1 | # ------------------------------------------------------------------- 2 | # 3 | # Copyright (c) 2014 Basho Technologies, Inc. 4 | # 5 | # This file is provided to you under the Apache License, 6 | # Version 2.0 (the "License"); you may not use this file 7 | # except in compliance with the License. You may obtain 8 | # a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | # ------------------------------------------------------------------- 20 | 21 | # ------------------------------------------------------------------- 22 | # NOTE: This file is is from https://github.com/basho/tools.mk. 23 | # It should not be edited in a project. It should simply be updated 24 | # wholesale when a new version of tools.mk is released. 25 | # ------------------------------------------------------------------- 26 | 27 | REBAR ?= ./rebar 28 | REVISION ?= $(shell git rev-parse --short HEAD) 29 | PROJECT ?= $(shell basename `find src -name "*.app.src"` .app.src) 30 | EUNIT_OPTS ?= 31 | 32 | .PHONY: compile-no-deps test docs xref dialyzer-run dialyzer-quick dialyzer \ 33 | cleanplt upload-docs 34 | 35 | compile-no-deps: 36 | ${REBAR} compile skip_deps=true 37 | 38 | test: ct eunit 39 | 40 | eunit: compile 41 | ${REBAR} ${EUNIT_OPTS} eunit skip_deps=true 42 | 43 | ct: compile 44 | ${REBAR} ct skip_deps=true 45 | 46 | upload-docs: docs 47 | @if [ -z "${BUCKET}" -o -z "${PROJECT}" -o -z "${REVISION}" ]; then \ 48 | echo "Set BUCKET, PROJECT, and REVISION env vars to upload docs"; \ 49 | exit 1; fi 50 | @cd doc; s3cmd put -P * "s3://${BUCKET}/${PROJECT}/${REVISION}/" > /dev/null 51 | @echo "Docs built at: http://${BUCKET}.s3-website-us-east-1.amazonaws.com/${PROJECT}/${REVISION}" 52 | 53 | docs: 54 | ${REBAR} doc skip_deps=true 55 | 56 | xref: compile 57 | ${REBAR} xref skip_deps=true 58 | 59 | PLT ?= $(HOME)/.combo_dialyzer_plt 60 | LOCAL_PLT = .local_dialyzer_plt 61 | DIALYZER_FLAGS ?= -Wunmatched_returns 62 | 63 | ${PLT}: compile 64 | @if [ -f $(PLT) ]; then \ 65 | dialyzer --check_plt --plt $(PLT) --apps $(DIALYZER_APPS) && \ 66 | dialyzer --add_to_plt --plt $(PLT) --output_plt $(PLT) --apps $(DIALYZER_APPS) ; test $$? -ne 1; \ 67 | else \ 68 | dialyzer --build_plt --output_plt $(PLT) --apps $(DIALYZER_APPS); test $$? -ne 1; \ 69 | fi 70 | 71 | ${LOCAL_PLT}: compile 72 | @if [ -d deps ]; then \ 73 | if [ -f $(LOCAL_PLT) ]; then \ 74 | dialyzer --check_plt --plt $(LOCAL_PLT) deps/*/ebin && \ 75 | dialyzer --add_to_plt --plt $(LOCAL_PLT) --output_plt $(LOCAL_PLT) deps/*/ebin ; test $$? -ne 1; \ 76 | else \ 77 | dialyzer --build_plt --output_plt $(LOCAL_PLT) deps/*/ebin ; test $$? -ne 1; \ 78 | fi \ 79 | fi 80 | 81 | dialyzer-run: 82 | @echo "==> $(shell basename $(shell pwd)) (dialyzer)" 83 | # The bulk of the code below deals with the dialyzer.ignore-warnings file 84 | # which contains strings to ignore if output by dialyzer. 85 | # Typically the strings include line numbers. Using them exactly is hard 86 | # to maintain as the code changes. This approach instead ignores the line 87 | # numbers, but takes into account the number of times a string is listed 88 | # for a given file. So if one string is listed once, for example, and it 89 | # appears twice in the warnings, the user is alerted. It is possible but 90 | # unlikely that this approach could mask a warning if one ignored warning 91 | # is removed and two warnings of the same kind appear in the file, for 92 | # example. But it is a trade-off that seems worth it. 93 | # Details of the cryptic commands: 94 | # - Remove line numbers from dialyzer.ignore-warnings 95 | # - Pre-pend duplicate count to each warning with sort | uniq -c 96 | # - Remove annoying white space around duplicate count 97 | # - Save in dialyer.ignore-warnings.tmp 98 | # - Do the same to dialyzer_warnings 99 | # - Remove matches from dialyzer.ignore-warnings.tmp from output 100 | # - Remove duplicate count 101 | # - Escape regex special chars to use lines as regex patterns 102 | # - Add pattern to match any line number (file.erl:\d+:) 103 | # - Anchor to match the entire line (^entire line$) 104 | # - Save in dialyzer_unhandled_warnings 105 | # - Output matches for those patterns found in the original warnings 106 | @if [ -f $(LOCAL_PLT) ]; then \ 107 | PLTS="$(PLT) $(LOCAL_PLT)"; \ 108 | else \ 109 | PLTS=$(PLT); \ 110 | fi; \ 111 | if [ -f dialyzer.ignore-warnings ]; then \ 112 | if [ $$(grep -cvE '[^[:space:]]' dialyzer.ignore-warnings) -ne 0 ]; then \ 113 | echo "ERROR: dialyzer.ignore-warnings contains a blank/empty line, this will match all messages!"; \ 114 | exit 1; \ 115 | fi; \ 116 | dialyzer $(DIALYZER_FLAGS) --plts $${PLTS} -c ebin > dialyzer_warnings ; \ 117 | cat dialyzer.ignore-warnings \ 118 | | sed -E 's/^([^:]+:)[^:]+:/\1/' \ 119 | | sort \ 120 | | uniq -c \ 121 | | sed -E '/.*\.erl: /!s/^[[:space:]]*[0-9]+[[:space:]]*//' \ 122 | > dialyzer.ignore-warnings.tmp ; \ 123 | egrep -v "^[[:space:]]*(done|Checking|Proceeding|Compiling)" dialyzer_warnings \ 124 | | sed -E 's/^([^:]+:)[^:]+:/\1/' \ 125 | | sort \ 126 | | uniq -c \ 127 | | sed -E '/.*\.erl: /!s/^[[:space:]]*[0-9]+[[:space:]]*//' \ 128 | | grep -F -f dialyzer.ignore-warnings.tmp -v \ 129 | | sed -E 's/^[[:space:]]*[0-9]+[[:space:]]*//' \ 130 | | sed -E 's/([]\^:+?|()*.$${}\[])/\\\1/g' \ 131 | | sed -E 's/(\\\.erl\\\:)/\1[[:digit:]]+:/g' \ 132 | | sed -E 's/^(.*)$$/^[[:space:]]*\1$$/g' \ 133 | > dialyzer_unhandled_warnings ; \ 134 | rm dialyzer.ignore-warnings.tmp; \ 135 | if [ $$(cat dialyzer_unhandled_warnings | wc -l) -gt 0 ]; then \ 136 | egrep -f dialyzer_unhandled_warnings dialyzer_warnings ; \ 137 | found_warnings=1; \ 138 | fi; \ 139 | [ "$$found_warnings" != 1 ] ; \ 140 | else \ 141 | dialyzer $(DIALYZER_FLAGS) --plts $${PLTS} -c ebin; \ 142 | fi 143 | 144 | dialyzer-quick: compile-no-deps dialyzer-run 145 | 146 | dialyzer: ${PLT} ${LOCAL_PLT} dialyzer-run 147 | 148 | cleanplt: 149 | @echo 150 | @echo "Are you sure? It takes several minutes to re-build." 151 | @echo Deleting $(PLT) and $(LOCAL_PLT) in 5 seconds. 152 | @echo 153 | sleep 5 154 | rm $(PLT) 155 | rm $(LOCAL_PLT) 156 | --------------------------------------------------------------------------------