├── .gitignore ├── LICENSE.txt ├── README.md ├── TODO.md ├── src ├── mango.app.src ├── mango.hrl ├── mango_crud.erl ├── mango_cursor.erl ├── mango_cursor.hrl ├── mango_cursor_text.erl ├── mango_cursor_view.erl ├── mango_doc.erl ├── mango_error.erl ├── mango_fields.erl ├── mango_httpd.erl ├── mango_idx.erl ├── mango_idx.hrl ├── mango_idx_special.erl ├── mango_idx_text.erl ├── mango_idx_view.erl ├── mango_json.erl ├── mango_native_proc.erl ├── mango_opts.erl ├── mango_selector.erl ├── mango_selector_text.erl ├── mango_sort.erl └── mango_util.erl └── test ├── 01-index-crud-test.py ├── 02-basic-find-test.py ├── 03-operator-test.py ├── 04-key-tests.py ├── 05-index-selection-test.py ├── 06-basic-text-test.py ├── 06-text-default-field-test.py ├── 07-text-custom-field-list-test.py ├── 08-text-limit-test.py ├── 09-text-sort-test.py ├── 10-disable-array-length-field-test.py ├── README.md ├── friend_docs.py ├── limit_docs.py ├── mango.py └── user_docs.py /.gitignore: -------------------------------------------------------------------------------- 1 | ebin/ 2 | test/*.pyc 3 | venv/ 4 | .eunit 5 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright 2014 IBM Corporation 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /TODO.md: -------------------------------------------------------------------------------- 1 | 2 | * Patch the view engine to do alternative sorts. This will include both the lower level couch\_view* modules as well as the fabric coordinators. 3 | 4 | * Patch the view engine so we can specify options when returning docs from cursors. We'll want this so that we can delete specific revisions from a document. 5 | 6 | * Need to figure out how to do raw collation on some indices because at 7 | least the _id index uses it forcefully. 8 | 9 | * Add lots more to the update API. Mongo appears to be missing some pretty obvious easy functionality here. Things like managing values doing things like multiplying numbers, or common string mutations would be obvious examples. Also it could be interesting to add to the language so that you can do conditional updates based on other document attributes. Definitely not a V1 endeavor. -------------------------------------------------------------------------------- /src/mango.app.src: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | {application, mango, [ 14 | {description, "MongoDB API compatibility layer for CouchDB"}, 15 | {vsn, git}, 16 | {registered, []}, 17 | {applications, [ 18 | kernel, 19 | stdlib, 20 | config, 21 | twig, 22 | fabric 23 | ]} 24 | ]}. 25 | -------------------------------------------------------------------------------- /src/mango.hrl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -define(MANGO_ERROR(R), throw({mango_error, ?MODULE, R})). 14 | -------------------------------------------------------------------------------- /src/mango_crud.erl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -module(mango_crud). 14 | 15 | -export([ 16 | insert/3, 17 | find/5, 18 | update/4, 19 | delete/3, 20 | explain/3 21 | ]). 22 | 23 | -export([ 24 | collect_cb/2, 25 | maybe_add_user_ctx/2 26 | ]). 27 | 28 | 29 | -include_lib("couch/include/couch_db.hrl"). 30 | -include("mango.hrl"). 31 | 32 | 33 | insert(Db, #doc{}=Doc, Opts) -> 34 | insert(Db, [Doc], Opts); 35 | insert(Db, {_}=Doc, Opts) -> 36 | insert(Db, [Doc], Opts); 37 | insert(Db, Docs, Opts0) when is_list(Docs) -> 38 | Opts1 = maybe_add_user_ctx(Db, Opts0), 39 | Opts2 = maybe_int_to_str(w, Opts1), 40 | case fabric:update_docs(Db, Docs, Opts2) of 41 | {ok, Results0} -> 42 | {ok, lists:zipwith(fun result_to_json/2, Docs, Results0)}; 43 | {accepted, Results0} -> 44 | {ok, lists:zipwith(fun result_to_json/2, Docs, Results0)}; 45 | {aborted, Errors} -> 46 | {error, lists:map(fun result_to_json/1, Errors)} 47 | end. 48 | 49 | 50 | find(Db, Selector, Callback, UserAcc, Opts0) -> 51 | Opts1 = maybe_add_user_ctx(Db, Opts0), 52 | Opts2 = maybe_int_to_str(r, Opts1), 53 | {ok, Cursor} = mango_cursor:create(Db, Selector, Opts2), 54 | mango_cursor:execute(Cursor, Callback, UserAcc). 55 | 56 | 57 | update(Db, Selector, Update, Options) -> 58 | Upsert = proplists:get_value(upsert, Options), 59 | case collect_docs(Db, Selector, Options) of 60 | {ok, []} when Upsert -> 61 | InitDoc = mango_doc:update_as_insert(Update), 62 | case mango_doc:has_operators(InitDoc) of 63 | true -> 64 | ?MANGO_ERROR(invalid_upsert_with_operators); 65 | false -> 66 | % Probably need to catch and rethrow errors from 67 | % this function. 68 | Doc = couch_doc:from_json_obj(InitDoc), 69 | NewDoc = case Doc#doc.id of 70 | <<"">> -> 71 | Doc#doc{id=couch_uuids:new(), revs={0, []}}; 72 | _ -> 73 | Doc 74 | end, 75 | insert(Db, NewDoc, Options) 76 | end; 77 | {ok, Docs} -> 78 | NewDocs = lists:map(fun(Doc) -> 79 | mango_doc:apply_update(Doc, Update) 80 | end, Docs), 81 | insert(Db, NewDocs, Options); 82 | Else -> 83 | Else 84 | end. 85 | 86 | 87 | delete(Db, Selector, Options) -> 88 | case collect_docs(Db, Selector, Options) of 89 | {ok, Docs} -> 90 | NewDocs = lists:map(fun({Props}) -> 91 | {[ 92 | {<<"_id">>, proplists:get_value(<<"_id">>, Props)}, 93 | {<<"_rev">>, proplists:get_value(<<"_rev">>, Props)}, 94 | {<<"_deleted">>, true} 95 | ]} 96 | end, Docs), 97 | insert(Db, NewDocs, Options); 98 | Else -> 99 | Else 100 | end. 101 | 102 | 103 | explain(Db, Selector, Opts0) -> 104 | Opts1 = maybe_add_user_ctx(Db, Opts0), 105 | Opts2 = maybe_int_to_str(r, Opts1), 106 | {ok, Cursor} = mango_cursor:create(Db, Selector, Opts2), 107 | mango_cursor:explain(Cursor). 108 | 109 | 110 | maybe_add_user_ctx(Db, Opts) -> 111 | case lists:keyfind(user_ctx, 1, Opts) of 112 | {user_ctx, _} -> 113 | Opts; 114 | false -> 115 | [{user_ctx, Db#db.user_ctx} | Opts] 116 | end. 117 | 118 | 119 | maybe_int_to_str(_Key, []) -> 120 | []; 121 | maybe_int_to_str(Key, [{Key, Val} | Rest]) when is_integer(Val) -> 122 | [{Key, integer_to_list(Val)} | maybe_int_to_str(Key, Rest)]; 123 | maybe_int_to_str(Key, [KV | Rest]) -> 124 | [KV | maybe_int_to_str(Key, Rest)]. 125 | 126 | 127 | result_to_json(#doc{id=Id}, Result) -> 128 | result_to_json(Id, Result); 129 | result_to_json({Props}, Result) -> 130 | Id = couch_util:get_value(<<"_id">>, Props), 131 | result_to_json(Id, Result); 132 | result_to_json(DocId, {ok, NewRev}) -> 133 | {[ 134 | {id, DocId}, 135 | {rev, couch_doc:rev_to_str(NewRev)} 136 | ]}; 137 | result_to_json(DocId, {accepted, NewRev}) -> 138 | {[ 139 | {id, DocId}, 140 | {rev, couch_doc:rev_to_str(NewRev)}, 141 | {accepted, true} 142 | ]}; 143 | result_to_json(DocId, Error) -> 144 | % chttpd:error_info/1 because this is coming from fabric 145 | % and not internal mango operations. 146 | {_Code, ErrorStr, Reason} = chttpd:error_info(Error), 147 | {[ 148 | {id, DocId}, 149 | {error, ErrorStr}, 150 | {reason, Reason} 151 | ]}. 152 | 153 | 154 | % This is for errors because for some reason we 155 | % need a different return value for errors? Blargh. 156 | result_to_json({{Id, Rev}, Error}) -> 157 | {_Code, ErrorStr, Reason} = chttpd:error_info(Error), 158 | {[ 159 | {id, Id}, 160 | {rev, couch_doc:rev_to_str(Rev)}, 161 | {error, ErrorStr}, 162 | {reason, Reason} 163 | ]}. 164 | 165 | 166 | collect_docs(Db, Selector, Options) -> 167 | Cb = fun ?MODULE:collect_cb/2, 168 | case find(Db, Selector, Cb, [], Options) of 169 | {ok, Docs} -> 170 | {ok, lists:reverse(Docs)}; 171 | Else -> 172 | Else 173 | end. 174 | 175 | 176 | collect_cb({row, Doc}, Acc) -> 177 | {ok, [Doc | Acc]}. 178 | 179 | -------------------------------------------------------------------------------- /src/mango_cursor.erl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -module(mango_cursor). 14 | 15 | 16 | -export([ 17 | create/3, 18 | explain/1, 19 | execute/3 20 | ]). 21 | 22 | 23 | -include_lib("couch/include/couch_db.hrl"). 24 | -include("mango.hrl"). 25 | -include("mango_cursor.hrl"). 26 | 27 | 28 | -define(SUPERVISOR, mango_cursor_sup). 29 | 30 | 31 | create(Db, Selector0, Opts) -> 32 | Selector = mango_selector:normalize(Selector0), 33 | 34 | ExistingIndexes = mango_idx:list(Db), 35 | if ExistingIndexes /= [] -> ok; true -> 36 | ?MANGO_ERROR({no_usable_index, no_indexes_defined}) 37 | end, 38 | 39 | FilteredIndexes = maybe_filter_indexes(ExistingIndexes, Opts), 40 | if FilteredIndexes /= [] -> ok; true -> 41 | ?MANGO_ERROR({no_usable_index, no_index_matching_name}) 42 | end, 43 | 44 | SortIndexes = mango_idx:for_sort(FilteredIndexes, Opts), 45 | if SortIndexes /= [] -> ok; true -> 46 | ?MANGO_ERROR({no_usable_index, missing_sort_index}) 47 | end, 48 | 49 | UsableFilter = fun(I) -> mango_idx:is_usable(I, Selector) end, 50 | UsableIndexes = lists:filter(UsableFilter, SortIndexes), 51 | if UsableIndexes /= [] -> ok; true -> 52 | ?MANGO_ERROR({no_usable_index, selector_unsupported}) 53 | end, 54 | 55 | create_cursor(Db, UsableIndexes, Selector, Opts). 56 | 57 | 58 | explain(#cursor{}=Cursor) -> 59 | #cursor{ 60 | index = Idx, 61 | selector = Selector, 62 | opts = Opts0, 63 | limit = Limit, 64 | skip = Skip, 65 | fields = Fields 66 | } = Cursor, 67 | Mod = mango_idx:cursor_mod(Idx), 68 | Opts = lists:keydelete(user_ctx, 1, Opts0), 69 | {[ 70 | {dbname, mango_idx:dbname(Idx)}, 71 | {index, mango_idx:to_json(Idx)}, 72 | {selector, Selector}, 73 | {opts, {Opts}}, 74 | {limit, Limit}, 75 | {skip, Skip}, 76 | {fields, Fields} 77 | ] ++ Mod:explain(Cursor)}. 78 | 79 | 80 | execute(#cursor{index=Idx}=Cursor, UserFun, UserAcc) -> 81 | Mod = mango_idx:cursor_mod(Idx), 82 | Mod:execute(Cursor, UserFun, UserAcc). 83 | 84 | 85 | maybe_filter_indexes(Indexes, Opts) -> 86 | case lists:keyfind(use_index, 1, Opts) of 87 | {use_index, []} -> 88 | Indexes; 89 | {use_index, [DesignId]} -> 90 | filter_indexes(Indexes, DesignId); 91 | {use_index, [DesignId, ViewName]} -> 92 | filter_indexes(Indexes, DesignId, ViewName) 93 | end. 94 | 95 | 96 | filter_indexes(Indexes, DesignId0) -> 97 | DesignId = case DesignId0 of 98 | <<"_design/", _/binary>> -> 99 | DesignId0; 100 | Else -> 101 | <<"_design/", Else/binary>> 102 | end, 103 | FiltFun = fun(I) -> mango_idx:ddoc(I) == DesignId end, 104 | lists:filter(FiltFun, Indexes). 105 | 106 | 107 | filter_indexes(Indexes0, DesignId, ViewName) -> 108 | Indexes = filter_indexes(Indexes0, DesignId), 109 | FiltFun = fun(I) -> mango_idx:name(I) == ViewName end, 110 | lists:filter(FiltFun, Indexes). 111 | 112 | 113 | create_cursor(Db, Indexes, Selector, Opts) -> 114 | [{CursorMod, CursorModIndexes} | _] = group_indexes_by_type(Indexes), 115 | CursorMod:create(Db, CursorModIndexes, Selector, Opts). 116 | 117 | 118 | group_indexes_by_type(Indexes) -> 119 | IdxDict = lists:foldl(fun(I, D) -> 120 | dict:append(mango_idx:cursor_mod(I), I, D) 121 | end, dict:new(), Indexes), 122 | % The first cursor module that has indexes will be 123 | % used to service this query. This is so that we 124 | % don't suddenly switch indexes for existing client 125 | % queries. 126 | CursorModules = [ 127 | mango_cursor_view, 128 | mango_cursor_text 129 | ], 130 | lists:flatmap(fun(CMod) -> 131 | case dict:find(CMod, IdxDict) of 132 | {ok, CModIndexes} -> 133 | [{CMod, CModIndexes}]; 134 | error -> 135 | [] 136 | end 137 | end, CursorModules). 138 | -------------------------------------------------------------------------------- /src/mango_cursor.hrl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -record(cursor, { 14 | db, 15 | index, 16 | ranges, 17 | selector, 18 | opts, 19 | limit = 10000000000, 20 | skip = 0, 21 | fields = undefined, 22 | user_fun, 23 | user_acc 24 | }). -------------------------------------------------------------------------------- /src/mango_cursor_text.erl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -module(mango_cursor_text). 14 | 15 | -export([ 16 | create/4, 17 | explain/1, 18 | execute/3 19 | ]). 20 | 21 | 22 | -include_lib("couch/include/couch_db.hrl"). 23 | -include_lib("dreyfus/include/dreyfus.hrl"). 24 | -include("mango_cursor.hrl"). 25 | -include("mango.hrl"). 26 | 27 | 28 | -record(cacc, { 29 | selector, 30 | dbname, 31 | ddocid, 32 | idx_name, 33 | query_args, 34 | bookmark, 35 | limit, 36 | skip, 37 | user_fun, 38 | user_acc, 39 | fields 40 | }). 41 | 42 | 43 | create(Db, Indexes, Selector, Opts0) -> 44 | Index = case Indexes of 45 | [Index0] -> 46 | Index0; 47 | _ -> 48 | ?MANGO_ERROR(multiple_text_indexes) 49 | end, 50 | 51 | Opts = unpack_bookmark(Db#db.name, Opts0), 52 | DreyfusLimit = get_dreyfus_limit(), 53 | Limit = erlang:min(DreyfusLimit, couch_util:get_value(limit, Opts, 50)), 54 | Skip = couch_util:get_value(skip, Opts, 0), 55 | Fields = couch_util:get_value(fields, Opts, all_fields), 56 | 57 | {ok, #cursor{ 58 | db = Db, 59 | index = Index, 60 | ranges = null, 61 | selector = Selector, 62 | opts = Opts, 63 | limit = Limit, 64 | skip = Skip, 65 | fields = Fields 66 | }}. 67 | 68 | 69 | explain(Cursor) -> 70 | #cursor{ 71 | selector = Selector, 72 | opts = Opts 73 | } = Cursor, 74 | [ 75 | {'query', mango_selector_text:convert(Selector)}, 76 | {sort, sort_query(Opts, Selector)} 77 | ]. 78 | 79 | 80 | execute(Cursor, UserFun, UserAcc) -> 81 | #cursor{ 82 | db = Db, 83 | index = Idx, 84 | limit = Limit, 85 | skip = Skip, 86 | selector = Selector, 87 | opts = Opts 88 | } = Cursor, 89 | QueryArgs = #index_query_args{ 90 | q = mango_selector_text:convert(Selector), 91 | sort = sort_query(Opts, Selector), 92 | raw_bookmark = true 93 | }, 94 | CAcc = #cacc{ 95 | selector = Selector, 96 | dbname = Db#db.name, 97 | ddocid = ddocid(Idx), 98 | idx_name = mango_idx:name(Idx), 99 | bookmark = get_bookmark(Opts), 100 | limit = Limit, 101 | skip = Skip, 102 | query_args = QueryArgs, 103 | user_fun = UserFun, 104 | user_acc = UserAcc, 105 | fields = Cursor#cursor.fields 106 | }, 107 | try 108 | execute(CAcc) 109 | catch 110 | throw:{stop, FinalCAcc} -> 111 | #cacc{ 112 | bookmark = FinalBM, 113 | user_fun = UserFun, 114 | user_acc = LastUserAcc 115 | } = FinalCAcc, 116 | JsonBM = dreyfus_bookmark:pack(FinalBM), 117 | Arg = {add_key, bookmark, JsonBM}, 118 | {_Go, FinalUserAcc} = UserFun(Arg, LastUserAcc), 119 | {ok, FinalUserAcc} 120 | end. 121 | 122 | 123 | execute(CAcc) -> 124 | case search_docs(CAcc) of 125 | {ok, Bookmark, []} -> 126 | % If we don't have any results from the 127 | % query it means the request has paged through 128 | % all possible results and the request is over. 129 | NewCAcc = CAcc#cacc{bookmark = Bookmark}, 130 | throw({stop, NewCAcc}); 131 | {ok, Bookmark, Hits} -> 132 | NewCAcc = CAcc#cacc{bookmark = Bookmark}, 133 | HitDocs = get_json_docs(CAcc#cacc.dbname, Hits), 134 | {ok, FinalCAcc} = handle_hits(NewCAcc, HitDocs), 135 | execute(FinalCAcc) 136 | end. 137 | 138 | 139 | search_docs(CAcc) -> 140 | #cacc{ 141 | dbname = DbName, 142 | ddocid = DDocId, 143 | idx_name = IdxName 144 | } = CAcc, 145 | QueryArgs = update_query_args(CAcc), 146 | case dreyfus_fabric_search:go(DbName, DDocId, IdxName, QueryArgs) of 147 | {ok, Bookmark, _, Hits, _, _} -> 148 | {ok, Bookmark, Hits}; 149 | {error, Reason} -> 150 | ?MANGO_ERROR({text_search_error, {error, Reason}}) 151 | end. 152 | 153 | 154 | handle_hits(CAcc, []) -> 155 | {ok, CAcc}; 156 | 157 | handle_hits(CAcc0, [{Sort, Doc} | Rest]) -> 158 | CAcc1 = handle_hit(CAcc0, Sort, Doc), 159 | handle_hits(CAcc1, Rest). 160 | 161 | 162 | handle_hit(CAcc0, Sort, Doc) -> 163 | #cacc{ 164 | limit = Limit, 165 | skip = Skip 166 | } = CAcc0, 167 | CAcc1 = update_bookmark(CAcc0, Sort), 168 | case mango_selector:match(CAcc1#cacc.selector, Doc) of 169 | true when Skip > 0 -> 170 | CAcc1#cacc{skip = Skip - 1}; 171 | true when Limit == 0 -> 172 | % We hit this case if the user spcified with a 173 | % zero limit. Notice that in this case we need 174 | % to return the bookmark from before this match 175 | throw({stop, CAcc0}); 176 | true when Limit == 1 -> 177 | NewCAcc = apply_user_fun(CAcc1, Doc), 178 | throw({stop, NewCAcc}); 179 | true when Limit > 1 -> 180 | NewCAcc = apply_user_fun(CAcc1, Doc), 181 | NewCAcc#cacc{limit = Limit - 1}; 182 | false -> 183 | CAcc1 184 | end. 185 | 186 | 187 | apply_user_fun(CAcc, Doc) -> 188 | FinalDoc = mango_fields:extract(Doc, CAcc#cacc.fields), 189 | #cacc{ 190 | user_fun = UserFun, 191 | user_acc = UserAcc 192 | } = CAcc, 193 | case UserFun({row, FinalDoc}, UserAcc) of 194 | {ok, NewUserAcc} -> 195 | CAcc#cacc{user_acc = NewUserAcc}; 196 | {stop, NewUserAcc} -> 197 | throw({stop, CAcc#cacc{user_acc = NewUserAcc}}) 198 | end. 199 | 200 | 201 | %% Convert Query to Dreyfus sort specifications 202 | %% Covert <<"Field">>, <<"desc">> to <<"-Field">> 203 | %% and append to the dreyfus query 204 | sort_query(Opts, Selector) -> 205 | {sort, {Sort}} = lists:keyfind(sort, 1, Opts), 206 | SortList = lists:map(fun(SortField) -> 207 | {Dir, RawSortField} = case SortField of 208 | {Field, <<"asc">>} -> {asc, Field}; 209 | {Field, <<"desc">>} -> {desc, Field}; 210 | Field when is_binary(Field) -> {asc, Field} 211 | end, 212 | SField = mango_selector_text:append_sort_type(RawSortField, Selector), 213 | case Dir of 214 | asc -> 215 | SField; 216 | desc -> 217 | <<"-", SField/binary>> 218 | end 219 | end, Sort), 220 | case SortList of 221 | [] -> relevance; 222 | _ -> SortList 223 | end. 224 | 225 | 226 | get_bookmark(Opts) -> 227 | case lists:keyfind(bookmark, 1, Opts) of 228 | {_, BM} when is_list(BM), BM /= [] -> 229 | BM; 230 | _ -> 231 | nil 232 | end. 233 | 234 | 235 | update_bookmark(CAcc, Sortable) -> 236 | BM = CAcc#cacc.bookmark, 237 | QueryArgs = CAcc#cacc.query_args, 238 | Sort = QueryArgs#index_query_args.sort, 239 | NewBM = dreyfus_bookmark:update(Sort, BM, [Sortable]), 240 | CAcc#cacc{bookmark = NewBM}. 241 | 242 | 243 | pack_bookmark(Bookmark) -> 244 | case dreyfus_bookmark:pack(Bookmark) of 245 | null -> nil; 246 | Enc -> Enc 247 | end. 248 | 249 | 250 | unpack_bookmark(DbName, Opts) -> 251 | NewBM = case lists:keyfind(bookmark, 1, Opts) of 252 | {_, nil} -> 253 | []; 254 | {_, Bin} -> 255 | try 256 | dreyfus_bookmark:unpack(DbName, Bin) 257 | catch _:_ -> 258 | ?MANGO_ERROR({invalid_bookmark, Bin}) 259 | end 260 | end, 261 | lists:keystore(bookmark, 1, Opts, {bookmark, NewBM}). 262 | 263 | 264 | ddocid(Idx) -> 265 | case mango_idx:ddoc(Idx) of 266 | <<"_design/", Rest/binary>> -> 267 | Rest; 268 | Else -> 269 | Else 270 | end. 271 | 272 | 273 | update_query_args(CAcc) -> 274 | #cacc{ 275 | bookmark = Bookmark, 276 | query_args = QueryArgs 277 | } = CAcc, 278 | QueryArgs#index_query_args{ 279 | bookmark = pack_bookmark(Bookmark), 280 | limit = get_limit(CAcc) 281 | }. 282 | 283 | 284 | get_limit(CAcc) -> 285 | erlang:min(get_dreyfus_limit(), CAcc#cacc.limit + CAcc#cacc.skip). 286 | 287 | 288 | get_dreyfus_limit() -> 289 | list_to_integer(config:get("dreyfus", "max_limit", "200")). 290 | 291 | 292 | get_json_docs(DbName, Hits) -> 293 | Ids = lists:map(fun(#sortable{item = Item}) -> 294 | couch_util:get_value(<<"_id">>, Item#hit.fields) 295 | end, Hits), 296 | {ok, IdDocs} = dreyfus_fabric:get_json_docs(DbName, Ids), 297 | lists:map(fun(#sortable{item = Item} = Sort) -> 298 | Id = couch_util:get_value(<<"_id">>, Item#hit.fields), 299 | case lists:keyfind(Id, 1, IdDocs) of 300 | {Id, {doc, Doc}} -> 301 | {Sort, Doc}; 302 | false -> 303 | {Sort, not_found} 304 | end 305 | end, Hits). 306 | 307 | -------------------------------------------------------------------------------- /src/mango_cursor_view.erl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -module(mango_cursor_view). 14 | 15 | -export([ 16 | create/4, 17 | explain/1, 18 | execute/3 19 | ]). 20 | 21 | -export([ 22 | handle_message/2 23 | ]). 24 | 25 | 26 | -include_lib("couch/include/couch_db.hrl"). 27 | -include("mango_cursor.hrl"). 28 | 29 | 30 | create(Db, Indexes, Selector, Opts) -> 31 | FieldRanges = mango_idx_view:field_ranges(Selector), 32 | Composited = composite_indexes(Indexes, FieldRanges), 33 | {Index, IndexRanges} = choose_best_index(Db, Composited), 34 | 35 | Limit = couch_util:get_value(limit, Opts, 10000000000), 36 | Skip = couch_util:get_value(skip, Opts, 0), 37 | Fields = couch_util:get_value(fields, Opts, all_fields), 38 | 39 | {ok, #cursor{ 40 | db = Db, 41 | index = Index, 42 | ranges = IndexRanges, 43 | selector = Selector, 44 | opts = Opts, 45 | limit = Limit, 46 | skip = Skip, 47 | fields = Fields 48 | }}. 49 | 50 | 51 | explain(Cursor) -> 52 | #cursor{ 53 | index = Idx, 54 | ranges = Ranges 55 | } = Cursor, 56 | case Ranges of 57 | [empty] -> 58 | [{range, empty}]; 59 | _ -> 60 | [{range, {[ 61 | {start_key, mango_idx:start_key(Idx, Ranges)}, 62 | {end_key, mango_idx:end_key(Idx, Ranges)} 63 | ]}}] 64 | end. 65 | 66 | 67 | execute(#cursor{db = Db, index = Idx} = Cursor0, UserFun, UserAcc) -> 68 | Cursor = Cursor0#cursor{ 69 | user_fun = UserFun, 70 | user_acc = UserAcc 71 | }, 72 | case Cursor#cursor.ranges of 73 | [empty] -> 74 | % empty indicates unsatisfiable ranges, so don't perform search 75 | {ok, UserAcc}; 76 | _ -> 77 | BaseArgs = #view_query_args{ 78 | view_type = red_map, 79 | start_key = mango_idx:start_key(Idx, Cursor#cursor.ranges), 80 | end_key = mango_idx:end_key(Idx, Cursor#cursor.ranges), 81 | include_docs = true 82 | }, 83 | Args = apply_opts(Cursor#cursor.opts, BaseArgs), 84 | CB = fun ?MODULE:handle_message/2, 85 | {ok, LastCursor} = case mango_idx:def(Idx) of 86 | all_docs -> 87 | fabric:all_docs(Db, CB, Cursor, Args); 88 | _ -> 89 | % Normal view 90 | DDoc = ddocid(Idx), 91 | Name = mango_idx:name(Idx), 92 | fabric:query_view(Db, DDoc, Name, CB, Cursor, Args) 93 | end, 94 | {ok, LastCursor#cursor.user_acc} 95 | end. 96 | 97 | 98 | % Any of these indexes may be a composite index. For each 99 | % index find the most specific set of fields for each 100 | % index. Ie, if an index has columns a, b, c, d, then 101 | % check FieldRanges for a, b, c, and d and return 102 | % the longest prefix of columns found. 103 | composite_indexes(Indexes, FieldRanges) -> 104 | lists:foldl(fun(Idx, Acc) -> 105 | Cols = mango_idx:columns(Idx), 106 | Prefix = composite_prefix(Cols, FieldRanges), 107 | [{Idx, Prefix} | Acc] 108 | end, [], Indexes). 109 | 110 | 111 | composite_prefix([], _) -> 112 | []; 113 | composite_prefix([Col | Rest], Ranges) -> 114 | case lists:keyfind(Col, 1, Ranges) of 115 | {Col, Range} -> 116 | [Range | composite_prefix(Rest, Ranges)]; 117 | false -> 118 | [] 119 | end. 120 | 121 | 122 | % Low and behold our query planner. Or something. 123 | % So stupid, but we can fix this up later. First 124 | % pass: Sort the IndexRanges by (num_columns, idx_name) 125 | % and return the first element. Yes. Its going to 126 | % be that dumb for now. 127 | % 128 | % In the future we can look into doing a cached parallel 129 | % reduce view read on each index with the ranges to find 130 | % the one that has the fewest number of rows or something. 131 | choose_best_index(_DbName, IndexRanges) -> 132 | Cmp = fun({A1, A2}, {B1, B2}) -> 133 | case length(A2) - length(B2) of 134 | N when N < 0 -> true; 135 | N when N == 0 -> 136 | % This is a really bad sort and will end 137 | % up preferring indices based on the 138 | % (dbname, ddocid, view_name) triple 139 | A1 =< B1; 140 | _ -> 141 | false 142 | end 143 | end, 144 | hd(lists:sort(Cmp, IndexRanges)). 145 | 146 | 147 | handle_message({total_and_offset, _, _} = _TO, Cursor) -> 148 | %twig:log(err, "TOTAL AND OFFSET: ~p", [_TO]), 149 | {ok, Cursor}; 150 | handle_message({row, {Props}}, Cursor) -> 151 | %twig:log(err, "ROW: ~p", [Props]), 152 | case doc_member(Cursor#cursor.db, Props, Cursor#cursor.opts) of 153 | {ok, Doc} -> 154 | case mango_selector:match(Cursor#cursor.selector, Doc) of 155 | true -> 156 | FinalDoc = mango_fields:extract(Doc, Cursor#cursor.fields), 157 | handle_doc(Cursor, FinalDoc); 158 | false -> 159 | {ok, Cursor} 160 | end; 161 | Error -> 162 | twig:log(err, "~s :: Error loading doc: ~p", [?MODULE, Error]), 163 | {ok, Cursor} 164 | end; 165 | handle_message(complete, Cursor) -> 166 | %twig:log(err, "COMPLETE", []), 167 | {ok, Cursor}; 168 | handle_message({error, Reason}, _Cursor) -> 169 | %twig:log(err, "ERROR: ~p", [Reason]), 170 | {error, Reason}. 171 | 172 | 173 | handle_doc(#cursor{skip = S} = C, _) when S > 0 -> 174 | {ok, C#cursor{skip = S - 1}}; 175 | handle_doc(#cursor{limit = L} = C, Doc) when L > 0 -> 176 | UserFun = C#cursor.user_fun, 177 | UserAcc = C#cursor.user_acc, 178 | {Go, NewAcc} = UserFun({row, Doc}, UserAcc), 179 | {Go, C#cursor{ 180 | user_acc = NewAcc, 181 | limit = L - 1 182 | }}; 183 | handle_doc(C, _Doc) -> 184 | {stop, C}. 185 | 186 | 187 | ddocid(Idx) -> 188 | case mango_idx:ddoc(Idx) of 189 | <<"_design/", Rest/binary>> -> 190 | Rest; 191 | Else -> 192 | Else 193 | end. 194 | 195 | 196 | apply_opts([], Args) -> 197 | Args; 198 | apply_opts([{r, RStr} | Rest], Args) -> 199 | IncludeDocs = case list_to_integer(RStr) of 200 | 1 -> 201 | true; 202 | R when R > 1 -> 203 | % We don't load the doc in the view query because 204 | % we have to do a quorum read in the coordinator 205 | % so there's no point. 206 | false 207 | end, 208 | NewArgs = Args#view_query_args{include_docs = IncludeDocs}, 209 | apply_opts(Rest, NewArgs); 210 | apply_opts([{conflicts, true} | Rest], Args) -> 211 | % I need to patch things so that views can specify 212 | % parameters when loading the docs from disk 213 | apply_opts(Rest, Args); 214 | apply_opts([{conflicts, false} | Rest], Args) -> 215 | % Ignored cause default 216 | apply_opts(Rest, Args); 217 | apply_opts([{sort, Sort} | Rest], Args) -> 218 | % We only support single direction sorts 219 | % so nothing fancy here. 220 | case mango_sort:directions(Sort) of 221 | [] -> 222 | apply_opts(Rest, Args); 223 | [<<"asc">> | _] -> 224 | apply_opts(Rest, Args); 225 | [<<"desc">> | _] -> 226 | SK = Args#view_query_args.start_key, 227 | SKDI = Args#view_query_args.start_docid, 228 | EK = Args#view_query_args.end_key, 229 | EKDI = Args#view_query_args.end_docid, 230 | NewArgs = Args#view_query_args{ 231 | direction = rev, 232 | start_key = EK, 233 | start_docid = EKDI, 234 | end_key = SK, 235 | end_docid = SKDI 236 | }, 237 | apply_opts(Rest, NewArgs) 238 | end; 239 | apply_opts([{_, _} | Rest], Args) -> 240 | % Ignore unknown options 241 | apply_opts(Rest, Args). 242 | 243 | 244 | doc_member(Db, RowProps, Opts) -> 245 | case couch_util:get_value(doc, RowProps) of 246 | {DocProps} -> 247 | {ok, {DocProps}}; 248 | undefined -> 249 | Id = couch_util:get_value(id, RowProps), 250 | case mango_util:defer(fabric, open_doc, [Db, Id, Opts]) of 251 | {ok, #doc{}=Doc} -> 252 | {ok, couch_doc:to_json_obj(Doc, [])}; 253 | Else -> 254 | Else 255 | end 256 | end. 257 | -------------------------------------------------------------------------------- /src/mango_error.erl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -module(mango_error). 14 | 15 | 16 | -include_lib("couch/include/couch_db.hrl"). 17 | 18 | 19 | -export([ 20 | info/2 21 | ]). 22 | 23 | 24 | info(mango_cursor, {no_usable_index, no_indexes_defined}) -> 25 | { 26 | 400, 27 | <<"no_usable_index">>, 28 | <<"There are no indexes defined in this database.">> 29 | }; 30 | info(mango_cursor, {no_usable_index, no_index_matching_name}) -> 31 | { 32 | 400, 33 | <<"no_usable_index">>, 34 | <<"No index matches the index specified with \"use_index\"">> 35 | }; 36 | info(mango_cursor, {no_usable_index, missing_sort_index}) -> 37 | { 38 | 400, 39 | <<"no_usable_index">>, 40 | <<"No index exists for this sort, try indexing by the sort fields.">> 41 | }; 42 | info(mango_cursor, {no_usable_index, selector_unsupported}) -> 43 | { 44 | 400, 45 | <<"no_usable_index">>, 46 | <<"There is no index available for this selector.">> 47 | }; 48 | 49 | info(mango_cursor_text, {invalid_bookmark, BadBookmark}) -> 50 | { 51 | 400, 52 | <<"invalid_bookmark">>, 53 | fmt("Invalid boomkark value: ~s", [?JSON_ENCODE(BadBookmark)]) 54 | }; 55 | info(mango_cursor_text, multiple_text_indexes) -> 56 | { 57 | 400, 58 | <<"multiple_text_indexes">>, 59 | <<"You must specify an index with the `use_index` parameter.">> 60 | }; 61 | info(mango_cursor_text, {text_search_error, {error, {bad_request, Msg}}}) 62 | when is_binary(Msg) -> 63 | { 64 | 400, 65 | <<"text_search_error">>, 66 | Msg 67 | }; 68 | info(mango_cursor_text, {text_search_error, {error, Error}}) -> 69 | { 70 | 400, 71 | <<"text_search_error">>, 72 | fmt("Error performing text search: ~p", [Error]) 73 | }; 74 | 75 | info(mango_fields, {invalid_fields_json, BadFields}) -> 76 | { 77 | 400, 78 | <<"invalid_fields">>, 79 | fmt("Fields must be an array of strings, not: ~w", [BadFields]) 80 | }; 81 | info(mango_fields, {invalid_field_json, BadField}) -> 82 | { 83 | 400, 84 | <<"invalid_field">>, 85 | fmt("Invalid JSON for field spec: ~w", [BadField]) 86 | }; 87 | 88 | info(mango_httpd, error_saving_ddoc) -> 89 | { 90 | 500, 91 | <<"error_saving_ddoc">>, 92 | <<"Unknown error while saving the design document.">> 93 | }; 94 | info(mango_httpd, {error_saving_ddoc, <<"conflict">>}) -> 95 | { 96 | 500, 97 | <<"error_saving_ddoc">>, 98 | <<"Encountered a conflict while saving the design document.">> 99 | }; 100 | info(mango_httpd, {error_saving_ddoc, Reason}) -> 101 | { 102 | 500, 103 | <<"error_saving_ddoc">>, 104 | fmt("Unknown error while saving the design document: ~s", [Reason]) 105 | }; 106 | 107 | info(mango_idx, {invalid_index_type, BadType}) -> 108 | { 109 | 400, 110 | <<"invalid_index">>, 111 | fmt("Invalid type for index: ~s", [BadType]) 112 | }; 113 | info(mango_idx, invalid_query_ddoc_language) -> 114 | { 115 | 400, 116 | <<"invalid_index">>, 117 | <<"Invalid design document query language.">> 118 | }; 119 | info(mango_idx, no_index_definition) -> 120 | { 121 | 400, 122 | <<"invalid_index">>, 123 | <<"Index is missing its definition.">> 124 | }; 125 | 126 | info(mango_idx_view, {invalid_index_json, BadIdx}) -> 127 | { 128 | 400, 129 | <<"invalid_index">>, 130 | fmt("JSON indexes must be an object, not: ~w", [BadIdx]) 131 | }; 132 | info(mango_idx_view, {index_not_found, BadIdx}) -> 133 | { 134 | 404, 135 | <<"invalid_index">>, 136 | fmt("JSON index ~s not found in this design doc.", [BadIdx]) 137 | }; 138 | 139 | info(mango_idx_text, {invalid_index_text, BadIdx}) -> 140 | { 141 | 400, 142 | <<"invalid_index">>, 143 | fmt("Text indexes must be an object, not: ~w", [BadIdx]) 144 | }; 145 | info(mango_idx_text, {invalid_index_fields_definition, Def}) -> 146 | { 147 | 400, 148 | <<"invalid_index_fields_definition">>, 149 | fmt("Text Index field definitions must be of the form 150 | {\"name\": \"fieldname\", \"type\": 151 | \"boolean,number, or string\"}. Def: ~p", [Def]) 152 | }; 153 | info(mango_idx_text, {index_not_found, BadIdx}) -> 154 | { 155 | 404, 156 | <<"index_not_found">>, 157 | fmt("Text index ~s not found in this design doc.", [BadIdx]) 158 | }; 159 | 160 | info(mango_opts, {invalid_ejson, Val}) -> 161 | { 162 | 400, 163 | <<"invalid_ejson">>, 164 | fmt("Invalid JSON value: ~w", [Val]) 165 | }; 166 | info(mango_opts, {invalid_key, Key}) -> 167 | { 168 | 400, 169 | <<"invalid_key">>, 170 | fmt("Invalid key ~s for this request.", [Key]) 171 | }; 172 | info(mango_opts, {missing_required_key, Key}) -> 173 | { 174 | 400, 175 | <<"missing_required_key">>, 176 | fmt("Missing required key: ~s", [Key]) 177 | }; 178 | info(mango_opts, {invalid_value, Name, Expect, Found}) -> 179 | { 180 | 400, 181 | <<"invalid_value">>, 182 | fmt("Value for ~s is ~w, should be ~w", [Name, Found, Expect]) 183 | }; 184 | info(mango_opts, {invalid_value, Name, Value}) -> 185 | { 186 | 400, 187 | <<"invalid_value">>, 188 | fmt("Invalid value for ~s: ~w", [Name, Value]) 189 | }; 190 | info(mango_opts, {invalid_string, Val}) -> 191 | { 192 | 400, 193 | <<"invalid_string">>, 194 | fmt("Invalid string: ~w", [Val]) 195 | }; 196 | info(mango_opts, {invalid_boolean, Val}) -> 197 | { 198 | 400, 199 | <<"invalid_boolean">>, 200 | fmt("Invalid boolean value: ~w", [Val]) 201 | }; 202 | info(mango_opts, {invalid_pos_integer, Val}) -> 203 | { 204 | 400, 205 | <<"invalid_pos_integer">>, 206 | fmt("~w is not an integer greater than zero", [Val]) 207 | }; 208 | info(mango_opts, {invalid_non_neg_integer, Val}) -> 209 | { 210 | 400, 211 | <<"invalid_non_neg_integer">>, 212 | fmt("~w is not an integer greater than or equal to zero", [Val]) 213 | }; 214 | info(mango_opts, {invalid_object, BadObj}) -> 215 | { 216 | 400, 217 | <<"invalid_object">>, 218 | fmt("~w is not a JSON object", [BadObj]) 219 | }; 220 | info(mango_opts, {invalid_selector_json, BadSel}) -> 221 | { 222 | 400, 223 | <<"invalid_selector_json">>, 224 | fmt("Selector must be a JSON object, not: ~w", [BadSel]) 225 | }; 226 | info(mango_opts, {invalid_index_name, BadName}) -> 227 | { 228 | 400, 229 | <<"invalid_index_name">>, 230 | fmt("Invalid index name: ~w", [BadName]) 231 | }; 232 | 233 | info(mango_opts, {multiple_text_operator, {invalid_selector, BadSel}}) -> 234 | { 235 | 400, 236 | <<"multiple_text_selector">>, 237 | fmt("Selector cannot contain more than one $text operator: ~w", 238 | [BadSel]) 239 | }; 240 | 241 | info(mango_selector, {invalid_selector, missing_field_name}) -> 242 | { 243 | 400, 244 | <<"invalid_selector">>, 245 | <<"One or more conditions is missing a field name.">> 246 | }; 247 | info(mango_selector, {bad_arg, Op, Arg}) -> 248 | { 249 | 400, 250 | <<"bad_arg">>, 251 | fmt("Bad argument for operator ~s: ~w", [Op, Arg]) 252 | }; 253 | info(mango_selector, {not_supported, Op}) -> 254 | { 255 | 400, 256 | <<"not_supported">>, 257 | fmt("Unsupported operator: ~s", [Op]) 258 | }; 259 | info(mango_selector, {invalid_operator, Op}) -> 260 | { 261 | 400, 262 | <<"invalid_operator">>, 263 | fmt("Invalid operator: ~s", [Op]) 264 | }; 265 | info(mango_selector, {bad_field, BadSel}) -> 266 | { 267 | 400, 268 | <<"bad_field">>, 269 | fmt("Invalid field normalization on selector: ~w", [BadSel]) 270 | }; 271 | 272 | info(mango_selector_text, {invalid_operator, Op}) -> 273 | { 274 | 400, 275 | <<"invalid_operator">>, 276 | fmt("Invalid text operator: ~s", [Op]) 277 | }; 278 | info(mango_selector_text, {text_sort_error, Field}) -> 279 | S = binary_to_list(Field), 280 | Msg = "Unspecified or ambiguous sort type. Try appending :number or" 281 | " :string to the sort field. ~s", 282 | { 283 | 400, 284 | <<"text_sort_error">>, 285 | fmt(Msg, [S]) 286 | }; 287 | 288 | info(mango_sort, {invalid_sort_json, BadSort}) -> 289 | { 290 | 400, 291 | <<"invalid_sort_json">>, 292 | fmt("Sort must be an array of sort specs, not: ~w", [BadSort]) 293 | }; 294 | info(mango_sort, {invalid_sort_dir, BadSpec}) -> 295 | { 296 | 400, 297 | <<"invalid_sort_dir">>, 298 | fmt("Invalid sort direction: ~w", BadSpec) 299 | }; 300 | info(mango_sort, {invalid_sort_field, BadField}) -> 301 | { 302 | 400, 303 | <<"invalid_sort_field">>, 304 | fmt("Invalid sort field: ~w", [BadField]) 305 | }; 306 | info(mango_sort, {unsupported, mixed_sort}) -> 307 | { 308 | 400, 309 | <<"unsupported_mixed_sort">>, 310 | <<"Sorts currently only support a single direction for all fields.">> 311 | }; 312 | 313 | info(mango_util, {error_loading_doc, DocId}) -> 314 | { 315 | 500, 316 | <<"internal_error">>, 317 | fmt("Error loading doc: ~s", [DocId]) 318 | }; 319 | info(mango_util, error_loading_ddocs) -> 320 | { 321 | 500, 322 | <<"internal_error">>, 323 | <<"Error loading design documents">> 324 | }; 325 | info(mango_util, {invalid_ddoc_lang, Lang}) -> 326 | { 327 | 400, 328 | <<"invalid_ddoc_lang">>, 329 | fmt("Existing design doc has an invalid language: ~w", [Lang]) 330 | }; 331 | 332 | info(Module, Reason) -> 333 | { 334 | 500, 335 | <<"unknown_error">>, 336 | fmt("Unknown Error: ~s :: ~w", [Module, Reason]) 337 | }. 338 | 339 | 340 | fmt(Format, Args) -> 341 | iolist_to_binary(io_lib:format(Format, Args)). 342 | -------------------------------------------------------------------------------- /src/mango_fields.erl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -module(mango_fields). 14 | 15 | -export([ 16 | new/1, 17 | extract/2 18 | ]). 19 | 20 | 21 | -include("mango.hrl"). 22 | 23 | 24 | new([]) -> 25 | {ok, all_fields}; 26 | new(Fields) when is_list(Fields) -> 27 | {ok, [field(F) || F <- Fields]}; 28 | new(Else) -> 29 | ?MANGO_ERROR({invalid_fields_json, Else}). 30 | 31 | 32 | extract(Doc, undefined) -> 33 | Doc; 34 | extract(Doc, all_fields) -> 35 | Doc; 36 | extract(Doc, Fields) -> 37 | lists:foldl(fun(F, NewDoc) -> 38 | {ok, Path} = mango_util:parse_field(F), 39 | case mango_doc:get_field(Doc, Path) of 40 | not_found -> 41 | NewDoc; 42 | bad_path -> 43 | NewDoc; 44 | Value -> 45 | mango_doc:set_field(NewDoc, Path, Value) 46 | end 47 | end, {[]}, Fields). 48 | 49 | 50 | field(Val) when is_binary(Val) -> 51 | Val; 52 | field({Val}) when is_list(Val) -> 53 | {Val}; 54 | field(Else) -> 55 | ?MANGO_ERROR({invalid_field_json, Else}). 56 | -------------------------------------------------------------------------------- /src/mango_httpd.erl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -module(mango_httpd). 14 | 15 | 16 | -export([ 17 | handle_req/2 18 | ]). 19 | 20 | 21 | -include_lib("couch/include/couch_db.hrl"). 22 | -include("mango.hrl"). 23 | 24 | 25 | handle_req(#httpd{} = Req, Db0) -> 26 | try 27 | Db = set_user_ctx(Req, Db0), 28 | handle_req_int(Req, Db) 29 | catch 30 | throw:{mango_error, Module, Reason} -> 31 | %Stack = erlang:get_stacktrace(), 32 | %twig:log(err, "Error: ~s :: ~w~n~p", [Module, Reason, Stack]), 33 | {Code, ErrorStr, ReasonStr} = mango_error:info(Module, Reason), 34 | Resp = {[ 35 | {<<"error">>, ErrorStr}, 36 | {<<"reason">>, ReasonStr} 37 | ]}, 38 | chttpd:send_json(Req, Code, Resp) 39 | end. 40 | 41 | 42 | handle_req_int(#httpd{path_parts=[_, <<"_index">> | _]} = Req, Db) -> 43 | handle_index_req(Req, Db); 44 | handle_req_int(#httpd{path_parts=[_, <<"_explain">> | _]} = Req, Db) -> 45 | handle_explain_req(Req, Db); 46 | handle_req_int(#httpd{path_parts=[_, <<"_find">> | _]} = Req, Db) -> 47 | handle_find_req(Req, Db); 48 | handle_req_int(_, _) -> 49 | throw({not_found, missing}). 50 | 51 | 52 | handle_index_req(#httpd{method='GET', path_parts=[_, _]}=Req, Db) -> 53 | Idxs = lists:sort(mango_idx:list(Db)), 54 | JsonIdxs = lists:map(fun mango_idx:to_json/1, Idxs), 55 | chttpd:send_json(Req, {[{indexes, JsonIdxs}]}); 56 | 57 | handle_index_req(#httpd{method='POST', path_parts=[_, _]}=Req, Db) -> 58 | {ok, Opts} = mango_opts:validate_idx_create(chttpd:json_body_obj(Req)), 59 | {ok, Idx0} = mango_idx:new(Db, Opts), 60 | {ok, Idx} = mango_idx:validate(Idx0), 61 | Id = mango_idx:ddoc(Idx), 62 | Name = mango_idx:name(Idx), 63 | {ok, DDoc} = mango_util:load_ddoc(Db, mango_idx:ddoc(Idx)), 64 | Status = case mango_idx:add(DDoc, Idx) of 65 | {ok, DDoc} -> 66 | <<"exists">>; 67 | {ok, NewDDoc} -> 68 | CreateOpts = get_idx_create_opts(Opts), 69 | case mango_crud:insert(Db, NewDDoc, CreateOpts) of 70 | {ok, [{RespProps}]} -> 71 | case lists:keyfind(error, 1, RespProps) of 72 | {error, Reason} -> 73 | ?MANGO_ERROR({error_saving_ddoc, Reason}); 74 | _ -> 75 | <<"created">> 76 | end; 77 | _ -> 78 | ?MANGO_ERROR(error_saving_ddoc) 79 | end 80 | end, 81 | chttpd:send_json(Req, {[{result, Status}, {id, Id}, {name, Name}]}); 82 | 83 | handle_index_req(#httpd{method='POST', path_parts=[_, <<"_index">>, 84 | <<"_bulk_delete">>]}=Req, Db) -> 85 | {ok, Opts} = mango_opts:validate_bulk_delete(chttpd:json_body_obj(Req)), 86 | DDocIds = get_bulk_delete_ddocs_ids(Opts), 87 | DelOpts = get_idx_create_opts(Opts), 88 | {Success, Error} = mango_idx:bulk_delete(Db, DDocIds, DelOpts), 89 | chttpd:send_json(Req, {[{<<"success">>, Success}, {<<"error">>, Error}]}); 90 | 91 | handle_index_req(#httpd{method='DELETE', 92 | path_parts=[A, B, <<"_design">>, DDocId0, Type, Name]}=Req, Db) -> 93 | PathParts = [A, B, <<"_design/", DDocId0/binary>>, Type, Name], 94 | handle_index_req(Req#httpd{path_parts=PathParts}, Db); 95 | 96 | handle_index_req(#httpd{method='DELETE', 97 | path_parts=[_, _, DDocId0, Type, Name]}=Req, Db) -> 98 | DDocId = case DDocId0 of 99 | <<"_design/", _/binary>> -> DDocId0; 100 | _ -> <<"_design/", DDocId0/binary>> 101 | end, 102 | Idxs = mango_idx:list(Db), 103 | Filt = fun(Idx) -> 104 | IsDDoc = mango_idx:ddoc(Idx) == DDocId, 105 | IsType = mango_idx:type(Idx) == Type, 106 | IsName = mango_idx:name(Idx) == Name, 107 | IsDDoc andalso IsType andalso IsName 108 | end, 109 | case lists:filter(Filt, Idxs) of 110 | [Idx] -> 111 | {ok, DDoc} = mango_util:load_ddoc(Db, mango_idx:ddoc(Idx)), 112 | {ok, NewDDoc} = mango_idx:remove(DDoc, Idx), 113 | FinalDDoc = case NewDDoc#doc.body of 114 | {[{<<"language">>, <<"query">>}]} -> 115 | NewDDoc#doc{deleted = true, body = {[]}}; 116 | _ -> 117 | NewDDoc 118 | end, 119 | DelOpts = get_idx_del_opts(Req), 120 | case mango_crud:insert(Db, FinalDDoc, DelOpts) of 121 | {ok, _} -> 122 | chttpd:send_json(Req, {[{ok, true}]}); 123 | _ -> 124 | ?MANGO_ERROR(error_saving_ddoc) 125 | end; 126 | [] -> 127 | throw({not_found, missing}) 128 | end; 129 | 130 | handle_index_req(Req, _Db) -> 131 | chttpd:send_method_not_allowed(Req, "GET,POST,DELETE"). 132 | 133 | 134 | handle_explain_req(#httpd{method='POST'}=Req, Db) -> 135 | {ok, Opts0} = mango_opts:validate_find(chttpd:json_body_obj(Req)), 136 | {value, {selector, Sel}, Opts} = lists:keytake(selector, 1, Opts0), 137 | Resp = mango_crud:explain(Db, Sel, Opts), 138 | chttpd:send_json(Req, Resp); 139 | 140 | handle_explain_req(Req, _Db) -> 141 | chttpd:send_method_not_allowed(Req, "POST"). 142 | 143 | 144 | handle_find_req(#httpd{method='POST'}=Req, Db) -> 145 | {ok, Opts0} = mango_opts:validate_find(chttpd:json_body_obj(Req)), 146 | {value, {selector, Sel}, Opts} = lists:keytake(selector, 1, Opts0), 147 | {ok, Resp0} = start_find_resp(Req), 148 | {ok, {Resp1, _, KVs}} = run_find(Resp0, Db, Sel, Opts), 149 | end_find_resp(Resp1, KVs); 150 | 151 | handle_find_req(Req, _Db) -> 152 | chttpd:send_method_not_allowed(Req, "POST"). 153 | 154 | 155 | set_user_ctx(#httpd{user_ctx=Ctx}, Db) -> 156 | Db#db{user_ctx=Ctx}. 157 | 158 | 159 | get_idx_create_opts(Opts) -> 160 | case lists:keyfind(w, 1, Opts) of 161 | {w, N} when is_integer(N), N > 0 -> 162 | [{w, integer_to_list(N)}]; 163 | _ -> 164 | [{w, "2"}] 165 | end. 166 | 167 | 168 | get_bulk_delete_ddocs_ids(Opts) -> 169 | case lists:keyfind(docids, 1, Opts) of 170 | {docids, DDocs} when is_list(DDocs) -> 171 | DDocs; 172 | _ -> 173 | [] 174 | end. 175 | 176 | 177 | get_idx_del_opts(Req) -> 178 | try 179 | WStr = chttpd:qs_value(Req, "w", "2"), 180 | _ = list_to_integer(WStr), 181 | [{w, WStr}] 182 | catch _:_ -> 183 | [{w, "2"}] 184 | end. 185 | 186 | 187 | start_find_resp(Req) -> 188 | chttpd:start_delayed_json_response(Req, 200, [], "{\"docs\":["). 189 | 190 | 191 | end_find_resp(Resp0, KVs) -> 192 | FinalAcc = lists:foldl(fun({K, V}, Acc) -> 193 | JK = ?JSON_ENCODE(K), 194 | JV = ?JSON_ENCODE(V), 195 | [JV, ": ", JK, ",\r\n" | Acc] 196 | end, ["\r\n]"], KVs), 197 | Chunk = lists:reverse(FinalAcc, ["}\r\n"]), 198 | {ok, Resp1} = chttpd:send_delayed_chunk(Resp0, Chunk), 199 | chttpd:end_delayed_json_response(Resp1). 200 | 201 | 202 | run_find(Resp, Db, Sel, Opts) -> 203 | mango_crud:find(Db, Sel, fun handle_doc/2, {Resp, "\r\n", []}, Opts). 204 | 205 | 206 | handle_doc({add_key, Key, Value}, {Resp, Prepend, KVs}) -> 207 | NewKVs = lists:keystore(Key, 1, KVs, {Key, Value}), 208 | {ok, {Resp, Prepend, NewKVs}}; 209 | handle_doc({row, Doc}, {Resp0, Prepend, KVs}) -> 210 | Chunk = [Prepend, ?JSON_ENCODE(Doc)], 211 | {ok, Resp1} = chttpd:send_delayed_chunk(Resp0, Chunk), 212 | {ok, {Resp1, ",\r\n", KVs}}. 213 | -------------------------------------------------------------------------------- /src/mango_idx.erl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | % This module is for the "index object" as in, the data structure 14 | % representing an index. Not to be confused with mango_index which 15 | % contains APIs for managing indexes. 16 | 17 | -module(mango_idx). 18 | 19 | 20 | -export([ 21 | list/1, 22 | recover/1, 23 | for_sort/2, 24 | 25 | new/2, 26 | validate/1, 27 | add/2, 28 | remove/2, 29 | bulk_delete/3, 30 | from_ddoc/2, 31 | special/1, 32 | 33 | dbname/1, 34 | ddoc/1, 35 | name/1, 36 | type/1, 37 | def/1, 38 | opts/1, 39 | columns/1, 40 | is_usable/2, 41 | start_key/2, 42 | end_key/2, 43 | cursor_mod/1, 44 | idx_mod/1, 45 | to_json/1 46 | ]). 47 | 48 | 49 | -include_lib("couch/include/couch_db.hrl"). 50 | -include("mango.hrl"). 51 | -include("mango_idx.hrl"). 52 | 53 | 54 | list(Db) -> 55 | {ok, Indexes} = ddoc_cache:open(db_to_name(Db), ?MODULE), 56 | Indexes. 57 | 58 | recover(Db) -> 59 | {ok, DDocs0} = mango_util:open_ddocs(Db), 60 | Pred = fun({Props}) -> 61 | case proplists:get_value(<<"language">>, Props) of 62 | <<"query">> -> true; 63 | _ -> false 64 | end 65 | end, 66 | DDocs = lists:filter(Pred, DDocs0), 67 | Special = special(Db), 68 | {ok, Special ++ lists:flatmap(fun(Doc) -> 69 | from_ddoc(Db, Doc) 70 | end, DDocs)}. 71 | 72 | 73 | for_sort(Indexes, Opts) -> 74 | % If a sort was specified we have to find an index that 75 | % can satisfy the request. 76 | case lists:keyfind(sort, 1, Opts) of 77 | {sort, {SProps}} when is_list(SProps) -> 78 | for_sort_int(Indexes, {SProps}); 79 | _ -> 80 | Indexes 81 | end. 82 | 83 | 84 | for_sort_int(Indexes, Sort) -> 85 | Fields = mango_sort:fields(Sort), 86 | FilterFun = fun(Idx) -> 87 | Cols = mango_idx:columns(Idx), 88 | case {mango_idx:type(Idx), Cols} of 89 | {_, all_fields} -> 90 | true; 91 | {<<"text">>, _} -> 92 | sets:is_subset(sets:from_list(Fields), sets:from_list(Cols)); 93 | {<<"json">>, _} -> 94 | lists:prefix(Fields, Cols); 95 | {<<"special">>, _} -> 96 | lists:prefix(Fields, Cols) 97 | end 98 | end, 99 | lists:filter(FilterFun, Indexes). 100 | 101 | 102 | new(Db, Opts) -> 103 | Def = get_idx_def(Opts), 104 | Type = get_idx_type(Opts), 105 | IdxName = get_idx_name(Def, Opts), 106 | DDoc = get_idx_ddoc(Def, Opts), 107 | {ok, #idx{ 108 | dbname = db_to_name(Db), 109 | ddoc = DDoc, 110 | name = IdxName, 111 | type = Type, 112 | def = Def, 113 | opts = filter_opts(Opts) 114 | }}. 115 | 116 | 117 | validate(Idx) -> 118 | Mod = idx_mod(Idx), 119 | Mod:validate(Idx). 120 | 121 | 122 | add(DDoc, Idx) -> 123 | Mod = idx_mod(Idx), 124 | {ok, NewDDoc} = Mod:add(DDoc, Idx), 125 | % Round trip through JSON for normalization 126 | Body = ?JSON_DECODE(?JSON_ENCODE(NewDDoc#doc.body)), 127 | {ok, NewDDoc#doc{body = Body}}. 128 | 129 | 130 | remove(DDoc, Idx) -> 131 | Mod = idx_mod(Idx), 132 | {ok, NewDDoc} = Mod:remove(DDoc, Idx), 133 | % Round trip through JSON for normalization 134 | Body = ?JSON_DECODE(?JSON_ENCODE(NewDDoc#doc.body)), 135 | {ok, NewDDoc#doc{body = Body}}. 136 | 137 | 138 | bulk_delete(Db, DDocIds, DelOpts0) -> 139 | DelOpts = mango_crud:maybe_add_user_ctx(Db, DelOpts0), 140 | {DeleteDocs, Errors} = lists:foldl(fun(DDocId0, {D, E}) -> 141 | Id = {<<"id">>, DDocId0}, 142 | case get_bulk_delete_ddoc(Db, DDocId0) of 143 | not_found -> 144 | {D, [{[Id, {<<"error">>, <<"does not exist">>}]} | E]}; 145 | invalid_ddoc_lang -> 146 | {D, [{[Id, {<<"error">>, <<"not a query doc">>}]} | E]}; 147 | error_loading_doc -> 148 | {D, [{[Id, {<<"error">>, <<"loading doc">>}]} | E]}; 149 | DDoc -> 150 | {[DDoc#doc{deleted = true, body = {[]}} | D], E } 151 | end 152 | end, {[], []}, DDocIds), 153 | case fabric:update_docs(Db, DeleteDocs, DelOpts) of 154 | {ok, Results} -> 155 | bulk_delete_results(lists:zip(DeleteDocs, Results), Errors); 156 | {accepted, Results} -> 157 | bulk_delete_results(lists:zip(DeleteDocs, Results), Errors); 158 | {aborted, Abort} -> 159 | bulk_delete_results(lists:zip(DeleteDocs, Abort), Errors) 160 | end. 161 | 162 | 163 | bulk_delete_results(DeleteResults, LoadErrors) -> 164 | {Success, Errors} = lists:foldl(fun({#doc{id=DDocId}, Result}, {S, E}) -> 165 | Id = {<<"id">>, DDocId}, 166 | case Result of 167 | {_, {_Pos, _}} -> 168 | {[{[Id, {<<"ok">>, true}]} | S], E}; 169 | {{_Id, _Rev}, Error} -> 170 | {_Code, ErrorStr, _Reason} = chttpd:error_info(Error), 171 | {S, [{[Id, {<<"error">>, ErrorStr}]} | E]}; 172 | Error -> 173 | {_Code, ErrorStr, _Reason} = chttpd:error_info(Error), 174 | {S, [{[Id, {<<"error">>, ErrorStr}]} | E]} 175 | end 176 | end, {[], []}, DeleteResults), 177 | {Success, Errors ++ LoadErrors}. 178 | 179 | 180 | get_bulk_delete_ddoc(Db, Id0) -> 181 | Id = case Id0 of 182 | <<"_design/", _/binary>> -> Id0; 183 | _ -> <<"_design/", Id0/binary>> 184 | end, 185 | try mango_util:open_doc(Db, Id) of 186 | {ok, #doc{deleted = false} = Doc} -> 187 | mango_util:check_lang(Doc), 188 | Doc; 189 | not_found -> 190 | not_found 191 | catch 192 | {{mango_error, mango_util, {invalid_ddoc_lang, _}}} -> 193 | invalid_ddoc_lang; 194 | {{mango_error, mango_util, {error_loading_doc, _}}} -> 195 | error_loading_doc 196 | end. 197 | 198 | 199 | from_ddoc(Db, {Props}) -> 200 | DbName = db_to_name(Db), 201 | DDoc = proplists:get_value(<<"_id">>, Props), 202 | 203 | case proplists:get_value(<<"language">>, Props) of 204 | <<"query">> -> ok; 205 | _ -> 206 | ?MANGO_ERROR(invalid_query_ddoc_language) 207 | end, 208 | 209 | IdxMods = [mango_idx_view, mango_idx_text], 210 | Idxs = lists:flatmap(fun(Mod) -> Mod:from_ddoc({Props}) end, IdxMods), 211 | lists:map(fun(Idx) -> 212 | Idx#idx{ 213 | dbname = DbName, 214 | ddoc = DDoc 215 | } 216 | end, Idxs). 217 | 218 | 219 | special(Db) -> 220 | AllDocs = #idx{ 221 | dbname = db_to_name(Db), 222 | name = <<"_all_docs">>, 223 | type = <<"special">>, 224 | def = all_docs, 225 | opts = [] 226 | }, 227 | % Add one for _update_seq 228 | [AllDocs]. 229 | 230 | 231 | dbname(#idx{dbname=DbName}) -> 232 | DbName. 233 | 234 | 235 | ddoc(#idx{ddoc=DDoc}) -> 236 | DDoc. 237 | 238 | 239 | name(#idx{name=Name}) -> 240 | Name. 241 | 242 | 243 | type(#idx{type=Type}) -> 244 | Type. 245 | 246 | 247 | def(#idx{def=Def}) -> 248 | Def. 249 | 250 | 251 | opts(#idx{opts=Opts}) -> 252 | Opts. 253 | 254 | 255 | to_json(#idx{}=Idx) -> 256 | Mod = idx_mod(Idx), 257 | Mod:to_json(Idx). 258 | 259 | 260 | columns(#idx{}=Idx) -> 261 | Mod = idx_mod(Idx), 262 | Mod:columns(Idx). 263 | 264 | 265 | is_usable(#idx{}=Idx, Selector) -> 266 | Mod = idx_mod(Idx), 267 | Mod:is_usable(Idx, Selector). 268 | 269 | 270 | start_key(#idx{}=Idx, Ranges) -> 271 | Mod = idx_mod(Idx), 272 | Mod:start_key(Ranges). 273 | 274 | 275 | end_key(#idx{}=Idx, Ranges) -> 276 | Mod = idx_mod(Idx), 277 | Mod:end_key(Ranges). 278 | 279 | 280 | cursor_mod(#idx{type = <<"json">>}) -> 281 | mango_cursor_view; 282 | cursor_mod(#idx{def = all_docs, type= <<"special">>}) -> 283 | mango_cursor_view; 284 | cursor_mod(#idx{type = <<"text">>}) -> 285 | mango_cursor_text. 286 | 287 | 288 | idx_mod(#idx{type = <<"json">>}) -> 289 | mango_idx_view; 290 | idx_mod(#idx{type = <<"special">>}) -> 291 | mango_idx_special; 292 | idx_mod(#idx{type = <<"text">>}) -> 293 | mango_idx_text. 294 | 295 | 296 | db_to_name(#db{name=Name}) -> 297 | Name; 298 | db_to_name(Name) when is_binary(Name) -> 299 | Name; 300 | db_to_name(Name) when is_list(Name) -> 301 | iolist_to_binary(Name). 302 | 303 | 304 | get_idx_def(Opts) -> 305 | case proplists:get_value(def, Opts) of 306 | undefined -> 307 | ?MANGO_ERROR(no_index_definition); 308 | Def -> 309 | Def 310 | end. 311 | 312 | 313 | get_idx_type(Opts) -> 314 | case proplists:get_value(type, Opts) of 315 | <<"json">> -> <<"json">>; 316 | <<"text">> -> <<"text">>; 317 | %<<"geo">> -> <<"geo">>; 318 | undefined -> <<"json">>; 319 | BadType -> 320 | ?MANGO_ERROR({invalid_index_type, BadType}) 321 | end. 322 | 323 | 324 | get_idx_ddoc(Idx, Opts) -> 325 | case proplists:get_value(ddoc, Opts) of 326 | <<"_design/", _Rest>> = Name -> 327 | Name; 328 | Name when is_binary(Name) -> 329 | <<"_design/", Name/binary>>; 330 | _ -> 331 | Bin = gen_name(Idx, Opts), 332 | <<"_design/", Bin/binary>> 333 | end. 334 | 335 | 336 | get_idx_name(Idx, Opts) -> 337 | case proplists:get_value(name, Opts) of 338 | Name when is_binary(Name) -> 339 | Name; 340 | _ -> 341 | gen_name(Idx, Opts) 342 | end. 343 | 344 | 345 | gen_name(Idx, Opts0) -> 346 | Opts = lists:usort(Opts0), 347 | TermBin = term_to_binary({Idx, Opts}), 348 | Sha = crypto:sha(TermBin), 349 | mango_util:enc_hex(Sha). 350 | 351 | 352 | filter_opts([]) -> 353 | []; 354 | filter_opts([{user_ctx, _} | Rest]) -> 355 | filter_opts(Rest); 356 | filter_opts([{ddoc, _} | Rest]) -> 357 | filter_opts(Rest); 358 | filter_opts([{name, _} | Rest]) -> 359 | filter_opts(Rest); 360 | filter_opts([{type, _} | Rest]) -> 361 | filter_opts(Rest); 362 | filter_opts([Opt | Rest]) -> 363 | [Opt | filter_opts(Rest)]. 364 | 365 | 366 | -------------------------------------------------------------------------------- /src/mango_idx.hrl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -record(idx, { 14 | dbname, 15 | ddoc, 16 | name, 17 | type, 18 | def, 19 | opts 20 | }). 21 | -------------------------------------------------------------------------------- /src/mango_idx_special.erl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -module(mango_idx_special). 14 | 15 | 16 | -export([ 17 | validate/1, 18 | add/2, 19 | remove/2, 20 | from_ddoc/1, 21 | to_json/1, 22 | columns/1, 23 | is_usable/2, 24 | start_key/1, 25 | end_key/1 26 | ]). 27 | 28 | 29 | -include_lib("couch/include/couch_db.hrl"). 30 | -include("mango_idx.hrl"). 31 | 32 | 33 | validate(_) -> 34 | erlang:exit(invalid_call). 35 | 36 | 37 | add(_, _) -> 38 | erlang:exit(invalid_call). 39 | 40 | 41 | remove(_, _) -> 42 | erlang:exit(invalid_call). 43 | 44 | 45 | from_ddoc(_) -> 46 | erlang:exit(invalid_call). 47 | 48 | 49 | to_json(#idx{def=all_docs}) -> 50 | {[ 51 | {ddoc, null}, 52 | {name, <<"_all_docs">>}, 53 | {type, <<"special">>}, 54 | {def, {[ 55 | {<<"fields">>, [{[ 56 | {<<"_id">>, <<"asc">>} 57 | ]}]} 58 | ]}} 59 | ]}. 60 | 61 | 62 | columns(#idx{def=all_docs}) -> 63 | [<<"_id">>]. 64 | 65 | 66 | is_usable(#idx{def=all_docs}, Selector) -> 67 | Fields = mango_idx_view:indexable_fields(Selector), 68 | lists:member(<<"_id">>, Fields). 69 | 70 | 71 | start_key([{'$gt', Key, _, _}]) -> 72 | case mango_json:special(Key) of 73 | true -> 74 | ?MIN_STR; 75 | false -> 76 | Key 77 | end; 78 | start_key([{'$gte', Key, _, _}]) -> 79 | false = mango_json:special(Key), 80 | Key; 81 | start_key([{'$eq', Key, '$eq', Key}]) -> 82 | false = mango_json:special(Key), 83 | Key. 84 | 85 | 86 | end_key([{_, _, '$lt', Key}]) -> 87 | case mango_json:special(Key) of 88 | true -> 89 | ?MAX_STR; 90 | false -> 91 | Key 92 | end; 93 | end_key([{_, _, '$lte', Key}]) -> 94 | false = mango_json:special(Key), 95 | Key; 96 | end_key([{'$eq', Key, '$eq', Key}]) -> 97 | false = mango_json:special(Key), 98 | Key. 99 | -------------------------------------------------------------------------------- /src/mango_idx_text.erl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -module(mango_idx_text). 14 | 15 | 16 | -export([ 17 | validate/1, 18 | validate_fields/1, 19 | add/2, 20 | remove/2, 21 | from_ddoc/1, 22 | to_json/1, 23 | columns/1, 24 | is_usable/2, 25 | get_default_field_options/1 26 | ]). 27 | 28 | 29 | -include_lib("couch/include/couch_db.hrl"). 30 | -include("mango.hrl"). 31 | -include("mango_idx.hrl"). 32 | 33 | 34 | validate(#idx{}=Idx) -> 35 | {ok, Def} = do_validate(Idx#idx.def), 36 | {ok, Idx#idx{def=Def}}. 37 | 38 | 39 | add(#doc{body={Props0}}=DDoc, Idx) -> 40 | Texts1 = case proplists:get_value(<<"indexes">>, Props0) of 41 | {Texts0} -> Texts0; 42 | _ -> [] 43 | end, 44 | NewText = make_text(Idx), 45 | Texts2 = lists:keystore(element(1, NewText), 1, Texts1, NewText), 46 | Props1 = lists:keystore(<<"indexes">>, 1, Props0, {<<"indexes">>, 47 | {Texts2}}), 48 | {ok, DDoc#doc{body={Props1}}}. 49 | 50 | 51 | remove(#doc{body={Props0}}=DDoc, Idx) -> 52 | Texts1 = case proplists:get_value(<<"indexes">>, Props0) of 53 | {Texts0} -> 54 | Texts0; 55 | _ -> 56 | ?MANGO_ERROR({index_not_found, Idx#idx.name}) 57 | end, 58 | Texts2 = lists:keydelete(Idx#idx.name, 1, Texts1), 59 | if Texts2 /= Texts1 -> ok; true -> 60 | ?MANGO_ERROR({index_not_found, Idx#idx.name}) 61 | end, 62 | Props1 = case Texts2 of 63 | [] -> 64 | lists:keydelete(<<"indexes">>, 1, Props0); 65 | _ -> 66 | lists:keystore(<<"indexes">>, 1, Props0, {<<"indexes">>, {Texts2}}) 67 | end, 68 | {ok, DDoc#doc{body={Props1}}}. 69 | 70 | 71 | from_ddoc({Props}) -> 72 | case lists:keyfind(<<"indexes">>, 1, Props) of 73 | {<<"indexes">>, {Texts}} when is_list(Texts) -> 74 | lists:flatmap(fun({Name, {VProps}}) -> 75 | Def = proplists:get_value(<<"index">>, VProps), 76 | I = #idx{ 77 | type = <<"text">>, 78 | name = Name, 79 | def = Def 80 | }, 81 | % TODO: Validate the index definition 82 | [I] 83 | end, Texts); 84 | _ -> 85 | [] 86 | end. 87 | 88 | 89 | to_json(Idx) -> 90 | {[ 91 | {ddoc, Idx#idx.ddoc}, 92 | {name, Idx#idx.name}, 93 | {type, Idx#idx.type}, 94 | {def, {def_to_json(Idx#idx.def)}} 95 | ]}. 96 | 97 | 98 | columns(Idx) -> 99 | {Props} = Idx#idx.def, 100 | {<<"fields">>, Fields} = lists:keyfind(<<"fields">>, 1, Props), 101 | case Fields of 102 | <<"all_fields">> -> 103 | all_fields; 104 | _ -> 105 | {DFProps} = couch_util:get_value(<<"default_field">>, Props, {[]}), 106 | Enabled = couch_util:get_value(<<"enabled">>, DFProps, true), 107 | Default = case Enabled of 108 | true -> [<<"$default">>]; 109 | false -> [] 110 | end, 111 | Default ++ lists:map(fun({FProps}) -> 112 | {_, Name} = lists:keyfind(<<"name">>, 1, FProps), 113 | {_, Type} = lists:keyfind(<<"type">>, 1, FProps), 114 | iolist_to_binary([Name, ":", Type]) 115 | end, Fields) 116 | end. 117 | 118 | 119 | is_usable(Idx, Selector) -> 120 | case columns(Idx) of 121 | all_fields -> 122 | true; 123 | Cols -> 124 | Fields = indexable_fields(Selector), 125 | sets:is_subset(sets:from_list(Fields), sets:from_list(Cols)) 126 | end. 127 | 128 | 129 | do_validate({Props}) -> 130 | {ok, Opts} = mango_opts:validate(Props, opts()), 131 | {ok, {Opts}}; 132 | do_validate(Else) -> 133 | ?MANGO_ERROR({invalid_index_text, Else}). 134 | 135 | 136 | def_to_json({Props}) -> 137 | def_to_json(Props); 138 | def_to_json([]) -> 139 | []; 140 | def_to_json([{<<"fields">>, <<"all_fields">>} | Rest]) -> 141 | [{<<"fields">>, []} | def_to_json(Rest)]; 142 | def_to_json([{fields, Fields} | Rest]) -> 143 | [{<<"fields">>, fields_to_json(Fields)} | def_to_json(Rest)]; 144 | def_to_json([{<<"fields">>, Fields} | Rest]) -> 145 | [{<<"fields">>, fields_to_json(Fields)} | def_to_json(Rest)]; 146 | def_to_json([{Key, Value} | Rest]) -> 147 | [{Key, Value} | def_to_json(Rest)]. 148 | 149 | 150 | fields_to_json([]) -> 151 | []; 152 | fields_to_json([{[{<<"name">>, Name}, {<<"type">>, Type0}]} | Rest]) -> 153 | Type = validate_field_type(Type0), 154 | [{[{Name, Type}]} | fields_to_json(Rest)]; 155 | fields_to_json([{[{<<"type">>, Type0}, {<<"name">>, Name}]} | Rest]) -> 156 | Type = validate_field_type(Type0), 157 | [{[{Name, Type}]} | fields_to_json(Rest)]. 158 | 159 | 160 | validate_field_type(<<"string">>) -> 161 | <<"string">>; 162 | validate_field_type(<<"number">>) -> 163 | <<"number">>; 164 | validate_field_type(<<"boolean">>) -> 165 | <<"boolean">>. 166 | 167 | 168 | validate_fields(Fields) -> 169 | try fields_to_json(Fields) of 170 | _ -> 171 | mango_fields:new(Fields) 172 | catch error:function_clause -> 173 | ?MANGO_ERROR({invalid_index_fields_definition, Fields}) 174 | end. 175 | 176 | 177 | opts() -> 178 | [ 179 | {<<"default_analyzer">>, [ 180 | {tag, default_analyzer}, 181 | {optional, true}, 182 | {default, <<"keyword">>} 183 | ]}, 184 | {<<"default_field">>, [ 185 | {tag, default_field}, 186 | {optional, true}, 187 | {default, {[]}} 188 | ]}, 189 | {<<"selector">>, [ 190 | {tag, selector}, 191 | {optional, true}, 192 | {default, {[]}}, 193 | {validator, fun mango_opts:validate_selector/1} 194 | ]}, 195 | {<<"fields">>, [ 196 | {tag, fields}, 197 | {optional, true}, 198 | {default, []}, 199 | {validator, fun ?MODULE:validate_fields/1} 200 | ]}, 201 | {<<"index_array_lengths">>, [ 202 | {tag, index_array_lengths}, 203 | {optional, true}, 204 | {default, true}, 205 | {validator, fun mango_opts:is_boolean/1} 206 | ]} 207 | ]. 208 | 209 | 210 | make_text(Idx) -> 211 | Text= {[ 212 | {<<"index">>, Idx#idx.def}, 213 | {<<"analyzer">>, construct_analyzer(Idx#idx.def)} 214 | ]}, 215 | {Idx#idx.name, Text}. 216 | 217 | 218 | get_default_field_options(Props) -> 219 | Default = couch_util:get_value(default_field, Props, {[]}), 220 | case Default of 221 | Bool when is_boolean(Bool) -> 222 | {Bool, <<"standard">>}; 223 | {[]} -> 224 | {true, <<"standard">>}; 225 | {Opts}-> 226 | Enabled = couch_util:get_value(<<"enabled">>, Opts, true), 227 | Analyzer = couch_util:get_value(<<"analyzer">>, Opts, 228 | <<"standard">>), 229 | {Enabled, Analyzer} 230 | end. 231 | 232 | 233 | construct_analyzer({Props}) -> 234 | DefaultAnalyzer = couch_util:get_value(default_analyzer, Props, 235 | <<"keyword">>), 236 | {DefaultField, DefaultFieldAnalyzer} = get_default_field_options(Props), 237 | DefaultAnalyzerDef = case DefaultField of 238 | true -> 239 | [{<<"$default">>, DefaultFieldAnalyzer}]; 240 | _ -> 241 | [] 242 | end, 243 | case DefaultAnalyzerDef of 244 | [] -> 245 | <<"keyword">>; 246 | _ -> 247 | {[ 248 | {<<"name">>, <<"perfield">>}, 249 | {<<"default">>, DefaultAnalyzer}, 250 | {<<"fields">>, {DefaultAnalyzerDef}} 251 | ]} 252 | end. 253 | 254 | 255 | indexable_fields(Selector) -> 256 | TupleTree = mango_selector_text:convert([], Selector), 257 | indexable_fields([], TupleTree). 258 | 259 | 260 | indexable_fields(Fields, {op_and, Args}) when is_list(Args) -> 261 | lists:foldl(fun(Arg, Fields0) -> indexable_fields(Fields0, Arg) end, 262 | Fields, Args); 263 | 264 | indexable_fields(Fields, {op_or, Args}) when is_list(Args) -> 265 | lists:foldl(fun(Arg, Fields0) -> indexable_fields(Fields0, Arg) end, 266 | Fields, Args); 267 | 268 | indexable_fields(Fields, {op_not, {ExistsQuery, Arg}}) when is_tuple(Arg) -> 269 | Fields0 = indexable_fields(Fields, ExistsQuery), 270 | indexable_fields(Fields0, Arg); 271 | 272 | indexable_fields(Fields, {op_insert, Arg}) when is_binary(Arg) -> 273 | Fields; 274 | 275 | indexable_fields(Fields, {op_field, {Name, _}}) -> 276 | [iolist_to_binary(Name) | Fields]; 277 | 278 | %% In this particular case, the lucene index is doing a field_exists query 279 | %% so it is looking at all sorts of combinations of field:* and field.* 280 | %% We don't add the field because we cannot pre-determine what field will exist. 281 | %% Hence we just return Fields and make it less restrictive. 282 | indexable_fields(Fields, {op_fieldname, {_, _}}) -> 283 | Fields; 284 | 285 | %% Similar idea to op_fieldname but with fieldname:null 286 | indexable_fields(Fields, {op_null, {_, _}}) -> 287 | Fields; 288 | 289 | indexable_fields(Fields, {op_default, _}) -> 290 | [<<"$default">> | Fields]. 291 | -------------------------------------------------------------------------------- /src/mango_idx_view.erl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -module(mango_idx_view). 14 | 15 | 16 | -export([ 17 | validate/1, 18 | add/2, 19 | remove/2, 20 | from_ddoc/1, 21 | to_json/1, 22 | is_usable/2, 23 | columns/1, 24 | start_key/1, 25 | end_key/1, 26 | 27 | indexable_fields/1, 28 | field_ranges/1, 29 | field_ranges/2 30 | ]). 31 | 32 | 33 | -include_lib("couch/include/couch_db.hrl"). 34 | -include("mango.hrl"). 35 | -include("mango_idx.hrl"). 36 | 37 | 38 | validate(#idx{}=Idx) -> 39 | {ok, Def} = do_validate(Idx#idx.def), 40 | {ok, Idx#idx{def=Def}}. 41 | 42 | 43 | add(#doc{body={Props0}}=DDoc, Idx) -> 44 | Views1 = case proplists:get_value(<<"views">>, Props0) of 45 | {Views0} -> Views0; 46 | _ -> [] 47 | end, 48 | NewView = make_view(Idx), 49 | Views2 = lists:keystore(element(1, NewView), 1, Views1, NewView), 50 | Props1 = lists:keystore(<<"views">>, 1, Props0, {<<"views">>, {Views2}}), 51 | {ok, DDoc#doc{body={Props1}}}. 52 | 53 | 54 | remove(#doc{body={Props0}}=DDoc, Idx) -> 55 | Views1 = case proplists:get_value(<<"views">>, Props0) of 56 | {Views0} -> 57 | Views0; 58 | _ -> 59 | ?MANGO_ERROR({index_not_found, Idx#idx.name}) 60 | end, 61 | Views2 = lists:keydelete(Idx#idx.name, 1, Views1), 62 | if Views2 /= Views1 -> ok; true -> 63 | ?MANGO_ERROR({index_not_found, Idx#idx.name}) 64 | end, 65 | Props1 = case Views2 of 66 | [] -> 67 | lists:keydelete(<<"views">>, 1, Props0); 68 | _ -> 69 | lists:keystore(<<"views">>, 1, Props0, {<<"views">>, {Views2}}) 70 | end, 71 | {ok, DDoc#doc{body={Props1}}}. 72 | 73 | 74 | from_ddoc({Props}) -> 75 | case lists:keyfind(<<"views">>, 1, Props) of 76 | {<<"views">>, {Views}} when is_list(Views) -> 77 | lists:flatmap(fun({Name, {VProps}}) -> 78 | Def = proplists:get_value(<<"map">>, VProps), 79 | {Opts0} = proplists:get_value(<<"options">>, VProps), 80 | Opts = lists:keydelete(<<"sort">>, 1, Opts0), 81 | I = #idx{ 82 | type = <<"json">>, 83 | name = Name, 84 | def = Def, 85 | opts = Opts 86 | }, 87 | % TODO: Validate the index definition 88 | [I] 89 | end, Views); 90 | _ -> 91 | [] 92 | end. 93 | 94 | 95 | to_json(Idx) -> 96 | {[ 97 | {ddoc, Idx#idx.ddoc}, 98 | {name, Idx#idx.name}, 99 | {type, Idx#idx.type}, 100 | {def, {def_to_json(Idx#idx.def)}} 101 | ]}. 102 | 103 | 104 | columns(Idx) -> 105 | {Props} = Idx#idx.def, 106 | {<<"fields">>, {Fields}} = lists:keyfind(<<"fields">>, 1, Props), 107 | [Key || {Key, _} <- Fields]. 108 | 109 | 110 | is_usable(Idx, Selector) -> 111 | % This index is usable if at least the first column is 112 | % a member of the indexable fields of the selector. 113 | Columns = columns(Idx), 114 | Fields = indexable_fields(Selector), 115 | lists:member(hd(Columns), Fields) and not is_text_search(Selector). 116 | 117 | 118 | is_text_search({[]}) -> 119 | false; 120 | is_text_search({[{<<"$default">>, _}]}) -> 121 | true; 122 | is_text_search({[{_Field, Cond}]}) when is_list(Cond) -> 123 | lists:foldl(fun(C, Exists) -> 124 | Exists or is_text_search(C) 125 | end, false, Cond); 126 | is_text_search({[{_Field, Cond}]}) when is_tuple(Cond) -> 127 | is_text_search(Cond); 128 | is_text_search({[{_Field, _Cond}]}) -> 129 | false; 130 | %% we reached values, which should always be false 131 | is_text_search(Val) 132 | when is_number(Val); is_boolean(Val); is_binary(Val)-> 133 | false. 134 | 135 | 136 | start_key([]) -> 137 | []; 138 | start_key([{'$gt', Key, _, _} | Rest]) -> 139 | case mango_json:special(Key) of 140 | true -> 141 | []; 142 | false -> 143 | [Key | start_key(Rest)] 144 | end; 145 | start_key([{'$gte', Key, _, _} | Rest]) -> 146 | false = mango_json:special(Key), 147 | [Key | start_key(Rest)]; 148 | start_key([{'$eq', Key, '$eq', Key} | Rest]) -> 149 | false = mango_json:special(Key), 150 | [Key | start_key(Rest)]. 151 | 152 | 153 | end_key([]) -> 154 | [{[]}]; 155 | end_key([{_, _, '$lt', Key} | Rest]) -> 156 | case mango_json:special(Key) of 157 | true -> 158 | [{[]}]; 159 | false -> 160 | [Key | end_key(Rest)] 161 | end; 162 | end_key([{_, _, '$lte', Key} | Rest]) -> 163 | false = mango_json:special(Key), 164 | [Key | end_key(Rest)]; 165 | end_key([{'$eq', Key, '$eq', Key} | Rest]) -> 166 | false = mango_json:special(Key), 167 | [Key | end_key(Rest)]. 168 | 169 | 170 | do_validate({Props}) -> 171 | {ok, Opts} = mango_opts:validate(Props, opts()), 172 | {ok, {Opts}}; 173 | do_validate(Else) -> 174 | ?MANGO_ERROR({invalid_index_json, Else}). 175 | 176 | 177 | def_to_json({Props}) -> 178 | def_to_json(Props); 179 | def_to_json([]) -> 180 | []; 181 | def_to_json([{fields, Fields} | Rest]) -> 182 | [{<<"fields">>, mango_sort:to_json(Fields)} | def_to_json(Rest)]; 183 | def_to_json([{<<"fields">>, Fields} | Rest]) -> 184 | [{<<"fields">>, mango_sort:to_json(Fields)} | def_to_json(Rest)]; 185 | def_to_json([{Key, Value} | Rest]) -> 186 | [{Key, Value} | def_to_json(Rest)]. 187 | 188 | 189 | opts() -> 190 | [ 191 | {<<"fields">>, [ 192 | {tag, fields}, 193 | {validator, fun mango_opts:validate_sort/1} 194 | ]} 195 | ]. 196 | 197 | 198 | make_view(Idx) -> 199 | View = {[ 200 | {<<"map">>, Idx#idx.def}, 201 | {<<"reduce">>, <<"_count">>}, 202 | {<<"options">>, {Idx#idx.opts}} 203 | ]}, 204 | {Idx#idx.name, View}. 205 | 206 | 207 | % This function returns a list of indexes that 208 | % can be used to restrict this query. This works by 209 | % searching the selector looking for field names that 210 | % can be "seen". 211 | % 212 | % Operators that can be seen through are '$and' and any of 213 | % the logical comparisons ('$lt', '$eq', etc). Things like 214 | % '$regex', '$in', '$nin', and '$or' can't be serviced by 215 | % a single index scan so we disallow them. In the future 216 | % we may become more clever and increase our ken such that 217 | % we will be able to see through these with crafty indexes 218 | % or new uses for existing indexes. For instance, I could 219 | % see an '$or' between comparisons on the same field becoming 220 | % the equivalent of a multi-query. But that's for another 221 | % day. 222 | 223 | % We can see through '$and' trivially 224 | indexable_fields({[{<<"$and">>, Args}]}) -> 225 | lists:usort(lists:flatten([indexable_fields(A) || A <- Args])); 226 | 227 | % So far we can't see through any other operator 228 | indexable_fields({[{<<"$", _/binary>>, _}]}) -> 229 | []; 230 | 231 | % If we have a field with a terminator that is locatable 232 | % using an index then the field is a possible index 233 | indexable_fields({[{Field, Cond}]}) -> 234 | case indexable(Cond) of 235 | true -> 236 | [Field]; 237 | false -> 238 | [] 239 | end; 240 | 241 | % An empty selector 242 | indexable_fields({[]}) -> 243 | []. 244 | 245 | 246 | % Check if a condition is indexable. The logical 247 | % comparisons are mostly straight forward. We 248 | % currently don't understand '$in' which is 249 | % theoretically supportable. '$nin' and '$ne' 250 | % aren't currently supported because they require 251 | % multiple index scans. 252 | indexable({[{<<"$lt">>, _}]}) -> 253 | true; 254 | indexable({[{<<"$lte">>, _}]}) -> 255 | true; 256 | indexable({[{<<"$eq">>, _}]}) -> 257 | true; 258 | indexable({[{<<"$gt">>, _}]}) -> 259 | true; 260 | indexable({[{<<"$gte">>, _}]}) -> 261 | true; 262 | 263 | % All other operators are currently not indexable. 264 | % This is also a subtle assertion that we don't 265 | % call indexable/1 on a field name. 266 | indexable({[{<<"$", _/binary>>, _}]}) -> 267 | false. 268 | 269 | 270 | % For each field, return {Field, Range} 271 | field_ranges(Selector) -> 272 | Fields = indexable_fields(Selector), 273 | field_ranges(Selector, Fields). 274 | 275 | 276 | field_ranges(Selector, Fields) -> 277 | field_ranges(Selector, Fields, []). 278 | 279 | 280 | field_ranges(_Selector, [], Acc) -> 281 | lists:reverse(Acc); 282 | field_ranges(Selector, [Field | Rest], Acc) -> 283 | case range(Selector, Field) of 284 | empty -> 285 | [{Field, empty}]; 286 | Range -> 287 | field_ranges(Selector, Rest, [{Field, Range} | Acc]) 288 | end. 289 | 290 | 291 | % Find the complete range for a given index in this 292 | % selector. This works by AND'ing logical comparisons 293 | % together so that we can define the start and end 294 | % keys for a given index. 295 | % 296 | % Selector must have been normalized before calling 297 | % this function. 298 | range(Selector, Index) -> 299 | range(Selector, Index, '$gt', mango_json:min(), '$lt', mango_json:max()). 300 | 301 | 302 | % Adjust Low and High based on values found for the 303 | % givend Index in Selector. 304 | range({[{<<"$and">>, Args}]}, Index, LCmp, Low, HCmp, High) -> 305 | lists:foldl(fun 306 | (Arg, {LC, L, HC, H}) -> 307 | range(Arg, Index, LC, L, HC, H); 308 | (_Arg, empty) -> 309 | empty 310 | end, {LCmp, Low, HCmp, High}, Args); 311 | 312 | % We can currently only traverse '$and' operators 313 | range({[{<<"$", _/binary>>}]}, _Index, LCmp, Low, HCmp, High) -> 314 | {LCmp, Low, HCmp, High}; 315 | 316 | % If the field name matches the index see if we can narrow 317 | % the acceptable range. 318 | range({[{Index, Cond}]}, Index, LCmp, Low, HCmp, High) -> 319 | range(Cond, LCmp, Low, HCmp, High); 320 | 321 | % Else we have a field unrelated to this index so just 322 | % return the current values. 323 | range(_, _, LCmp, Low, HCmp, High) -> 324 | {LCmp, Low, HCmp, High}. 325 | 326 | 327 | % The comments below are a bit cryptic at first but they show 328 | % where the Arg cand land in the current range. 329 | % 330 | % For instance, given: 331 | % 332 | % {$lt: N} 333 | % Low = 1 334 | % High = 5 335 | % 336 | % Depending on the value of N we can have one of five locations 337 | % in regards to a given Low/High pair: 338 | % 339 | % min low mid high max 340 | % 341 | % That is: 342 | % min = (N < Low) 343 | % low = (N == Low) 344 | % mid = (Low < N < High) 345 | % high = (N == High) 346 | % max = (High < N) 347 | % 348 | % If N < 1, (min) then the effective range is empty. 349 | % 350 | % If N == 1, (low) then we have to set the range to empty because 351 | % N < 1 && N >= 1 is an empty set. If the operator had been '$lte' 352 | % and LCmp was '$gte' or '$eq' then we could keep around the equality 353 | % check on Arg by setting LCmp == HCmp = '$eq' and Low == High == Arg. 354 | % 355 | % If 1 < N < 5 (mid), then we set High to Arg and Arg has just 356 | % narrowed our range. HCmp is set the the '$lt' operator that was 357 | % part of the input. 358 | % 359 | % If N == 5 (high), We just set HCmp to '$lt' since its guaranteed 360 | % to be equally or more restrictive than the current possible values 361 | % of '$lt' or '$lte'. 362 | % 363 | % If N > 5 (max), nothing changes as our current range is already 364 | % more narrow than the current condition. 365 | % 366 | % Obviously all of that logic gets tweaked for the other logical 367 | % operators but its all straight forward once you figure out how 368 | % we're basically just narrowing our logical ranges. 369 | 370 | range({[{<<"$lt">>, Arg}]}, LCmp, Low, HCmp, High) -> 371 | case range_pos(Low, Arg, High) of 372 | min -> 373 | empty; 374 | low -> 375 | empty; 376 | mid -> 377 | {LCmp, Low, '$lt', Arg}; 378 | high -> 379 | {LCmp, Low, '$lt', Arg}; 380 | max -> 381 | {LCmp, Low, HCmp, High} 382 | end; 383 | 384 | range({[{<<"$lte">>, Arg}]}, LCmp, Low, HCmp, High) -> 385 | case range_pos(Low, Arg, High) of 386 | min -> 387 | empty; 388 | low when LCmp == '$gte'; LCmp == '$eq' -> 389 | {'$eq', Arg, '$eq', Arg}; 390 | low -> 391 | empty; 392 | mid -> 393 | {LCmp, Low, '$lte', Arg}; 394 | high -> 395 | {LCmp, Low, HCmp, High}; 396 | max -> 397 | {LCmp, Low, HCmp, High} 398 | end; 399 | 400 | range({[{<<"$eq">>, Arg}]}, LCmp, Low, HCmp, High) -> 401 | case range_pos(Low, Arg, High) of 402 | min -> 403 | empty; 404 | low when LCmp == '$gte'; LCmp == '$eq' -> 405 | {'$eq', Arg, '$eq', Arg}; 406 | low -> 407 | empty; 408 | mid -> 409 | {'$eq', Arg, '$eq', Arg}; 410 | high when HCmp == '$lte'; HCmp == '$eq' -> 411 | {'$eq', Arg, '$eq', Arg}; 412 | high -> 413 | empty; 414 | max -> 415 | empty 416 | end; 417 | 418 | range({[{<<"$gte">>, Arg}]}, LCmp, Low, HCmp, High) -> 419 | case range_pos(Low, Arg, High) of 420 | min -> 421 | {LCmp, Low, HCmp, High}; 422 | low -> 423 | {LCmp, Low, HCmp, High}; 424 | mid -> 425 | {'$gte', Arg, HCmp, High}; 426 | high when HCmp == '$lte'; HCmp == '$eq' -> 427 | {'$eq', Arg, '$eq', Arg}; 428 | high -> 429 | empty; 430 | max -> 431 | empty 432 | end; 433 | 434 | range({[{<<"$gt">>, Arg}]}, LCmp, Low, HCmp, High) -> 435 | case range_pos(Low, Arg, High) of 436 | min -> 437 | {LCmp, Low, HCmp, High}; 438 | low -> 439 | {'$gt', Arg, HCmp, High}; 440 | mid -> 441 | {'$gt', Arg, HCmp, High}; 442 | high -> 443 | empty; 444 | max -> 445 | empty 446 | end; 447 | 448 | % There's some other un-indexable restriction on the index 449 | % that will be applied as a post-filter. Ignore it and 450 | % carry on our merry way. 451 | range({[{<<"$", _/binary>>, _}]}, LCmp, Low, HCmp, High) -> 452 | {LCmp, Low, HCmp, High}. 453 | 454 | 455 | % Returns the value min | low | mid | high | max depending 456 | % on how Arg compares to Low and High. 457 | range_pos(Low, Arg, High) -> 458 | case mango_json:cmp(Arg, Low) of 459 | N when N < 0 -> min; 460 | N when N == 0 -> low; 461 | _ -> 462 | case mango_json:cmp(Arg, High) of 463 | X when X < 0 -> 464 | mid; 465 | X when X == 0 -> 466 | high; 467 | _ -> 468 | max 469 | end 470 | end. 471 | -------------------------------------------------------------------------------- /src/mango_json.erl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -module(mango_json). 14 | 15 | 16 | -export([ 17 | min/0, 18 | max/0, 19 | cmp/2, 20 | cmp_raw/2, 21 | type/1, 22 | special/1, 23 | to_binary/1 24 | ]). 25 | 26 | 27 | -define(MIN_VAL, mango_json_min). 28 | -define(MAX_VAL, mango_json_max). 29 | 30 | 31 | min() -> 32 | ?MIN_VAL. 33 | 34 | 35 | max() -> 36 | ?MAX_VAL. 37 | 38 | 39 | cmp(?MIN_VAL, ?MIN_VAL) -> 40 | 0; 41 | cmp(?MIN_VAL, _) -> 42 | -1; 43 | cmp(_, ?MIN_VAL) -> 44 | 1; 45 | cmp(?MAX_VAL, ?MAX_VAL) -> 46 | 0; 47 | cmp(?MAX_VAL, _) -> 48 | 1; 49 | cmp(_, ?MAX_VAL) -> 50 | -1; 51 | cmp(A, B) -> 52 | couch_view:cmp_json(A, B). 53 | 54 | 55 | cmp_raw(?MIN_VAL, ?MIN_VAL) -> 56 | 0; 57 | cmp_raw(?MIN_VAL, _) -> 58 | -1; 59 | cmp_raw(_, ?MIN_VAL) -> 60 | 1; 61 | cmp_raw(?MAX_VAL, ?MAX_VAL) -> 62 | 0; 63 | cmp_raw(?MAX_VAL, _) -> 64 | 1; 65 | cmp_raw(_, ?MAX_VAL) -> 66 | -1; 67 | cmp_raw(A, B) -> 68 | case A < B of 69 | true -> 70 | -1; 71 | false -> 72 | case A > B of 73 | true -> 74 | 1; 75 | false -> 76 | 0 77 | end 78 | end. 79 | 80 | 81 | type(null) -> 82 | <<"null">>; 83 | type(Bool) when is_boolean(Bool) -> 84 | <<"boolean">>; 85 | type(Num) when is_number(Num) -> 86 | <<"number">>; 87 | type(Str) when is_binary(Str) -> 88 | <<"string">>; 89 | type({Props}) when is_list(Props) -> 90 | <<"object">>; 91 | type(Vals) when is_list(Vals) -> 92 | <<"array">>. 93 | 94 | 95 | special(?MIN_VAL) -> 96 | true; 97 | special(?MAX_VAL) -> 98 | true; 99 | special(_) -> 100 | false. 101 | 102 | 103 | to_binary({Props}) -> 104 | Pred = fun({Key, Value}) -> 105 | {to_binary(Key), to_binary(Value)} 106 | end, 107 | {lists:map(Pred, Props)}; 108 | to_binary(Data) when is_list(Data) -> 109 | [to_binary(D) || D <- Data]; 110 | to_binary(null) -> 111 | null; 112 | to_binary(true) -> 113 | true; 114 | to_binary(false) -> 115 | false; 116 | to_binary(Data) when is_atom(Data) -> 117 | list_to_binary(atom_to_list(Data)); 118 | to_binary(Data) when is_number(Data) -> 119 | Data; 120 | to_binary(Data) when is_binary(Data) -> 121 | Data. -------------------------------------------------------------------------------- /src/mango_native_proc.erl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -module(mango_native_proc). 14 | -behavior(gen_server). 15 | 16 | 17 | -export([ 18 | start_link/0, 19 | set_timeout/2, 20 | prompt/2 21 | ]). 22 | 23 | -export([ 24 | init/1, 25 | terminate/2, 26 | handle_call/3, 27 | handle_cast/2, 28 | handle_info/2, 29 | code_change/3 30 | ]). 31 | 32 | 33 | -record(st, { 34 | indexes = [], 35 | timeout = 5000 36 | }). 37 | 38 | 39 | -record(tacc, { 40 | index_array_lengths = true, 41 | fields = all_fields, 42 | path = [] 43 | }). 44 | 45 | 46 | start_link() -> 47 | gen_server:start_link(?MODULE, [], []). 48 | 49 | 50 | set_timeout(Pid, TimeOut) when is_integer(TimeOut), TimeOut > 0 -> 51 | gen_server:call(Pid, {set_timeout, TimeOut}). 52 | 53 | 54 | prompt(Pid, Data) -> 55 | gen_server:call(Pid, {prompt, Data}). 56 | 57 | 58 | init(_) -> 59 | {ok, #st{}}. 60 | 61 | 62 | terminate(_Reason, _St) -> 63 | ok. 64 | 65 | 66 | handle_call({set_timeout, TimeOut}, _From, St) -> 67 | {reply, ok, St#st{timeout=TimeOut}}; 68 | 69 | handle_call({prompt, [<<"reset">>]}, _From, St) -> 70 | {reply, true, St#st{indexes=[]}}; 71 | 72 | handle_call({prompt, [<<"reset">>, _QueryConfig]}, _From, St) -> 73 | {reply, true, St#st{indexes=[]}}; 74 | 75 | handle_call({prompt, [<<"add_fun">>, IndexInfo]}, _From, St) -> 76 | Indexes = St#st.indexes ++ [IndexInfo], 77 | NewSt = St#st{indexes = Indexes}, 78 | {reply, true, NewSt}; 79 | 80 | handle_call({prompt, [<<"map_doc">>, Doc]}, _From, St) -> 81 | {reply, map_doc(St, mango_json:to_binary(Doc)), St}; 82 | 83 | handle_call({prompt, [<<"reduce">>, _, _]}, _From, St) -> 84 | {reply, null, St}; 85 | 86 | handle_call({prompt, [<<"rereduce">>, _, _]}, _From, St) -> 87 | {reply, null, St}; 88 | 89 | handle_call({prompt, [<<"index_doc">>, Doc]}, _From, St) -> 90 | {reply, index_doc(St, mango_json:to_binary(Doc)), St}; 91 | 92 | handle_call(Msg, _From, St) -> 93 | {stop, {invalid_call, Msg}, {invalid_call, Msg}, St}. 94 | 95 | 96 | handle_cast(garbage_collect, St) -> 97 | erlang:garbage_collect(), 98 | {noreply, St}; 99 | 100 | handle_cast(Msg, St) -> 101 | {stop, {invalid_cast, Msg}, St}. 102 | 103 | 104 | handle_info(Msg, St) -> 105 | {stop, {invalid_info, Msg}, St}. 106 | 107 | 108 | code_change(_OldVsn, St, _Extra) -> 109 | {ok, St}. 110 | 111 | 112 | map_doc(#st{indexes=Indexes}, Doc) -> 113 | lists:map(fun(Idx) -> get_index_entries(Idx, Doc) end, Indexes). 114 | 115 | 116 | index_doc(#st{indexes=Indexes}, Doc) -> 117 | lists:map(fun(Idx) -> get_text_entries(Idx, Doc) end, Indexes). 118 | 119 | 120 | get_index_entries({IdxProps}, Doc) -> 121 | {Fields} = couch_util:get_value(<<"fields">>, IdxProps), 122 | Values = lists:map(fun({Field, _Dir}) -> 123 | case mango_doc:get_field(Doc, Field) of 124 | not_found -> not_found; 125 | bad_path -> not_found; 126 | Else -> Else 127 | end 128 | end, Fields), 129 | case lists:member(not_found, Values) of 130 | true -> 131 | []; 132 | false -> 133 | [[Values, null]] 134 | end. 135 | 136 | 137 | get_text_entries({IdxProps}, Doc) -> 138 | Selector = case couch_util:get_value(<<"selector">>, IdxProps) of 139 | [] -> {[]}; 140 | Else -> Else 141 | end, 142 | case should_index(Selector, Doc) of 143 | true -> 144 | get_text_entries0(IdxProps, Doc); 145 | false -> 146 | [] 147 | end. 148 | 149 | 150 | get_text_entries0(IdxProps, Doc) -> 151 | DefaultEnabled = get_default_enabled(IdxProps), 152 | IndexArrayLengths = get_index_array_lengths(IdxProps), 153 | FieldsList = get_text_field_list(IdxProps), 154 | TAcc = #tacc{ 155 | index_array_lengths = IndexArrayLengths, 156 | fields = FieldsList 157 | }, 158 | Fields0 = get_text_field_values(Doc, TAcc), 159 | Fields = if not DefaultEnabled -> Fields0; true -> 160 | add_default_text_field(Fields0) 161 | end, 162 | FieldNames = get_field_names(Fields, []), 163 | Converted = convert_text_fields(Fields), 164 | FieldNames ++ Converted. 165 | 166 | 167 | get_text_field_values({Props}, TAcc) when is_list(Props) -> 168 | get_text_field_values_obj(Props, TAcc, []); 169 | 170 | get_text_field_values(Values, TAcc) when is_list(Values) -> 171 | IndexArrayLengths = TAcc#tacc.index_array_lengths, 172 | NewPath = ["[]" | TAcc#tacc.path], 173 | NewTAcc = TAcc#tacc{path = NewPath}, 174 | case IndexArrayLengths of 175 | true -> 176 | % We bypass make_text_field and directly call make_text_field_name 177 | % because the length field name is not part of the path. 178 | LengthFieldName = make_text_field_name(NewTAcc#tacc.path, <<"length">>), 179 | LengthField = [{LengthFieldName, <<"length">>, length(Values)}], 180 | get_text_field_values_arr(Values, NewTAcc, LengthField); 181 | _ -> 182 | get_text_field_values_arr(Values, NewTAcc, []) 183 | end; 184 | 185 | get_text_field_values(Bin, TAcc) when is_binary(Bin) -> 186 | make_text_field(TAcc, <<"string">>, Bin); 187 | 188 | get_text_field_values(Num, TAcc) when is_number(Num) -> 189 | make_text_field(TAcc, <<"number">>, Num); 190 | 191 | get_text_field_values(Bool, TAcc) when is_boolean(Bool) -> 192 | make_text_field(TAcc, <<"boolean">>, Bool); 193 | 194 | get_text_field_values(null, TAcc) -> 195 | make_text_field(TAcc, <<"null">>, true). 196 | 197 | 198 | get_text_field_values_obj([], _, FAcc) -> 199 | FAcc; 200 | get_text_field_values_obj([{Key, Val} | Rest], TAcc, FAcc) -> 201 | NewPath = [Key | TAcc#tacc.path], 202 | NewTAcc = TAcc#tacc{path = NewPath}, 203 | Fields = get_text_field_values(Val, NewTAcc), 204 | get_text_field_values_obj(Rest, TAcc, Fields ++ FAcc). 205 | 206 | 207 | get_text_field_values_arr([], _, FAcc) -> 208 | FAcc; 209 | get_text_field_values_arr([Value | Rest], TAcc, FAcc) -> 210 | Fields = get_text_field_values(Value, TAcc), 211 | get_text_field_values_arr(Rest, TAcc, Fields ++ FAcc). 212 | 213 | 214 | get_default_enabled(Props) -> 215 | case couch_util:get_value(<<"default_field">>, Props, {[]}) of 216 | Bool when is_boolean(Bool) -> 217 | Bool; 218 | {[]} -> 219 | true; 220 | {Opts}-> 221 | couch_util:get_value(<<"enabled">>, Opts, true) 222 | end. 223 | 224 | 225 | get_index_array_lengths(Props) -> 226 | couch_util:get_value(<<"index_array_lengths">>, Props, true). 227 | 228 | 229 | add_default_text_field(Fields) -> 230 | DefaultFields = add_default_text_field(Fields, []), 231 | DefaultFields ++ Fields. 232 | 233 | 234 | add_default_text_field([], Acc) -> 235 | Acc; 236 | add_default_text_field([{_Name, <<"string">>, Value} | Rest], Acc) -> 237 | NewAcc = [{<<"$default">>, <<"string">>, Value} | Acc], 238 | add_default_text_field(Rest, NewAcc); 239 | add_default_text_field([_ | Rest], Acc) -> 240 | add_default_text_field(Rest, Acc). 241 | 242 | 243 | %% index of all field names 244 | get_field_names([], FAcc) -> 245 | FAcc; 246 | get_field_names([{Name, _Type, _Value} | Rest], FAcc) -> 247 | case lists:member([<<"$fieldnames">>, Name, []], FAcc) of 248 | true -> 249 | get_field_names(Rest, FAcc); 250 | false -> 251 | get_field_names(Rest, [[<<"$fieldnames">>, Name, []] | FAcc]) 252 | end. 253 | 254 | 255 | convert_text_fields([]) -> 256 | []; 257 | convert_text_fields([{Name, _Type, Value} | Rest]) -> 258 | [[Name, Value, []] | convert_text_fields(Rest)]. 259 | 260 | 261 | should_index(Selector, Doc) -> 262 | % We should do this 263 | NormSelector = mango_selector:normalize(Selector), 264 | Matches = mango_selector:match(NormSelector, Doc), 265 | IsDesign = case mango_doc:get_field(Doc, <<"_id">>) of 266 | <<"_design/", _/binary>> -> true; 267 | _ -> false 268 | end, 269 | Matches and not IsDesign. 270 | 271 | 272 | get_text_field_list(IdxProps) -> 273 | case couch_util:get_value(<<"fields">>, IdxProps) of 274 | Fields when is_list(Fields) -> 275 | RawList = lists:flatmap(fun get_text_field_info/1, Fields), 276 | [mango_util:lucene_escape_user(Field) || Field <- RawList]; 277 | _ -> 278 | all_fields 279 | end. 280 | 281 | 282 | get_text_field_info({Props}) -> 283 | Name = couch_util:get_value(<<"name">>, Props), 284 | Type0 = couch_util:get_value(<<"type">>, Props), 285 | if not is_binary(Name) -> []; true -> 286 | Type = get_text_field_type(Type0), 287 | [iolist_to_binary([Name, ":", Type])] 288 | end. 289 | 290 | 291 | get_text_field_type(<<"number">>) -> 292 | <<"number">>; 293 | get_text_field_type(<<"boolean">>) -> 294 | <<"boolean">>; 295 | get_text_field_type(_) -> 296 | <<"string">>. 297 | 298 | 299 | make_text_field(TAcc, Type, Value) -> 300 | FieldName = make_text_field_name(TAcc#tacc.path, Type), 301 | Fields = TAcc#tacc.fields, 302 | case Fields == all_fields orelse lists:member(FieldName, Fields) of 303 | true -> 304 | [{FieldName, Type, Value}]; 305 | false -> 306 | [] 307 | end. 308 | 309 | 310 | make_text_field_name([P | Rest], Type) -> 311 | Parts = lists:reverse(Rest, [iolist_to_binary([P, ":", Type])]), 312 | Escaped = [mango_util:lucene_escape_field(N) || N <- Parts], 313 | iolist_to_binary(mango_util:join(".", Escaped)). 314 | -------------------------------------------------------------------------------- /src/mango_opts.erl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -module(mango_opts). 14 | 15 | -export([ 16 | validate_idx_create/1, 17 | validate_find/1, 18 | validate_bulk_delete/1 19 | ]). 20 | 21 | -export([ 22 | validate/2, 23 | 24 | is_string/1, 25 | is_boolean/1, 26 | is_pos_integer/1, 27 | is_non_neg_integer/1, 28 | is_object/1, 29 | 30 | validate_idx_name/1, 31 | validate_selector/1, 32 | validate_use_index/1, 33 | validate_bookmark/1, 34 | validate_sort/1, 35 | validate_fields/1 36 | ]). 37 | 38 | 39 | -include("mango.hrl"). 40 | 41 | 42 | validate_idx_create({Props}) -> 43 | Opts = [ 44 | {<<"index">>, [ 45 | {tag, def} 46 | ]}, 47 | {<<"type">>, [ 48 | {tag, type}, 49 | {optional, true}, 50 | {default, <<"json">>}, 51 | {validator, fun is_string/1} 52 | ]}, 53 | {<<"name">>, [ 54 | {tag, name}, 55 | {optional, true}, 56 | {default, auto_name}, 57 | {validator, fun validate_idx_name/1} 58 | ]}, 59 | {<<"ddoc">>, [ 60 | {tag, ddoc}, 61 | {optional, true}, 62 | {default, auto_name}, 63 | {validator, fun validate_idx_name/1} 64 | ]}, 65 | {<<"w">>, [ 66 | {tag, w}, 67 | {optional, true}, 68 | {default, 2}, 69 | {validator, fun is_pos_integer/1} 70 | ]} 71 | ], 72 | validate(Props, Opts). 73 | 74 | 75 | validate_find({Props}) -> 76 | Opts = [ 77 | {<<"selector">>, [ 78 | {tag, selector}, 79 | {validator, fun validate_selector/1} 80 | ]}, 81 | {<<"use_index">>, [ 82 | {tag, use_index}, 83 | {optional, true}, 84 | {default, []}, 85 | {validator, fun validate_use_index/1} 86 | ]}, 87 | {<<"bookmark">>, [ 88 | {tag, bookmark}, 89 | {optional, true}, 90 | {default, <<>>}, 91 | {validator, fun validate_bookmark/1} 92 | ]}, 93 | {<<"limit">>, [ 94 | {tag, limit}, 95 | {optional, true}, 96 | {default, 10000000000}, 97 | {validator, fun is_non_neg_integer/1} 98 | ]}, 99 | {<<"skip">>, [ 100 | {tag, skip}, 101 | {optional, true}, 102 | {default, 0}, 103 | {validator, fun is_non_neg_integer/1} 104 | ]}, 105 | {<<"sort">>, [ 106 | {tag, sort}, 107 | {optional, true}, 108 | {default, []}, 109 | {validator, fun validate_sort/1} 110 | ]}, 111 | {<<"fields">>, [ 112 | {tag, fields}, 113 | {optional, true}, 114 | {default, []}, 115 | {validator, fun validate_fields/1} 116 | ]}, 117 | {<<"r">>, [ 118 | {tag, r}, 119 | {optional, true}, 120 | {default, 1}, 121 | {validator, fun mango_opts:is_pos_integer/1} 122 | ]}, 123 | {<<"conflicts">>, [ 124 | {tag, conflicts}, 125 | {optional, true}, 126 | {default, false}, 127 | {validator, fun mango_opts:is_boolean/1} 128 | ]} 129 | ], 130 | validate(Props, Opts). 131 | 132 | 133 | validate_bulk_delete({Props}) -> 134 | Opts = [ 135 | {<<"docids">>, [ 136 | {tag, docids}, 137 | {validator, fun validate_bulk_docs/1} 138 | ]}, 139 | {<<"w">>, [ 140 | {tag, w}, 141 | {optional, true}, 142 | {default, 2}, 143 | {validator, fun is_pos_integer/1} 144 | ]} 145 | ], 146 | validate(Props, Opts). 147 | 148 | 149 | validate(Props, Opts) -> 150 | case mango_util:assert_ejson({Props}) of 151 | true -> 152 | ok; 153 | false -> 154 | ?MANGO_ERROR({invalid_ejson, {Props}}) 155 | end, 156 | {Rest, Acc} = validate_opts(Opts, Props, []), 157 | case Rest of 158 | [] -> 159 | ok; 160 | [{BadKey, _} | _] -> 161 | ?MANGO_ERROR({invalid_key, BadKey}) 162 | end, 163 | {ok, Acc}. 164 | 165 | 166 | is_string(Val) when is_binary(Val) -> 167 | {ok, Val}; 168 | is_string(Else) -> 169 | ?MANGO_ERROR({invalid_string, Else}). 170 | 171 | 172 | is_boolean(true) -> 173 | {ok, true}; 174 | is_boolean(false) -> 175 | {ok, false}; 176 | is_boolean(Else) -> 177 | ?MANGO_ERROR({invalid_boolean, Else}). 178 | 179 | 180 | is_pos_integer(V) when is_integer(V), V > 0 -> 181 | {ok, V}; 182 | is_pos_integer(Else) -> 183 | ?MANGO_ERROR({invalid_pos_integer, Else}). 184 | 185 | 186 | is_non_neg_integer(V) when is_integer(V), V >= 0 -> 187 | {ok, V}; 188 | is_non_neg_integer(Else) -> 189 | ?MANGO_ERROR({invalid_non_neg_integer, Else}). 190 | 191 | 192 | is_object({Props}) -> 193 | true = mango_util:assert_ejson({Props}), 194 | {ok, {Props}}; 195 | is_object(Else) -> 196 | ?MANGO_ERROR({invalid_object, Else}). 197 | 198 | 199 | validate_idx_name(auto_name) -> 200 | {ok, auto_name}; 201 | validate_idx_name(Else) -> 202 | is_string(Else). 203 | 204 | 205 | validate_selector({Props}) -> 206 | Norm = mango_selector:normalize({Props}), 207 | {ok, Norm}; 208 | validate_selector(Else) -> 209 | ?MANGO_ERROR({invalid_selector_json, Else}). 210 | 211 | 212 | validate_bulk_docs(Docs) when is_list(Docs) -> 213 | lists:foreach(fun ?MODULE:is_string/1, Docs), 214 | {ok, Docs}; 215 | validate_bulk_docs(Else) -> 216 | ?MANGO_ERROR({invalid_bulk_docs, Else}). 217 | 218 | 219 | validate_use_index(IndexName) when is_binary(IndexName) -> 220 | case binary:split(IndexName, <<"/">>) of 221 | [DesignId] -> 222 | {ok, [DesignId]}; 223 | [<<"_design">>, DesignId] -> 224 | {ok, [DesignId]}; 225 | [DesignId, ViewName] -> 226 | {ok, [DesignId, ViewName]}; 227 | [<<"_design">>, DesignId, ViewName] -> 228 | {ok, [DesignId, ViewName]}; 229 | _ -> 230 | ?MANGO_ERROR({invalid_index_name, IndexName}) 231 | end; 232 | validate_use_index(null) -> 233 | {ok, []}; 234 | validate_use_index([]) -> 235 | {ok, []}; 236 | validate_use_index([DesignId]) when is_binary(DesignId) -> 237 | {ok, [DesignId]}; 238 | validate_use_index([DesignId, ViewName]) 239 | when is_binary(DesignId), is_binary(ViewName) -> 240 | {ok, [DesignId, ViewName]}; 241 | validate_use_index(Else) -> 242 | ?MANGO_ERROR({invalid_index_name, Else}). 243 | 244 | 245 | validate_bookmark(null) -> 246 | {ok, nil}; 247 | validate_bookmark(<<>>) -> 248 | {ok, nil}; 249 | validate_bookmark(Bin) when is_binary(Bin) -> 250 | {ok, Bin}; 251 | validate_bookmark(Else) -> 252 | ?MANGO_ERROR({invalid_bookmark, Else}). 253 | 254 | 255 | validate_sort(Value) -> 256 | mango_sort:new(Value). 257 | 258 | 259 | validate_fields(Value) -> 260 | mango_fields:new(Value). 261 | 262 | 263 | validate_opts([], Props, Acc) -> 264 | {Props, lists:reverse(Acc)}; 265 | validate_opts([{Name, Desc} | Rest], Props, Acc) -> 266 | {tag, Tag} = lists:keyfind(tag, 1, Desc), 267 | case lists:keytake(Name, 1, Props) of 268 | {value, {Name, Prop}, RestProps} -> 269 | NewAcc = [{Tag, validate_opt(Name, Desc, Prop)} | Acc], 270 | validate_opts(Rest, RestProps, NewAcc); 271 | false -> 272 | NewAcc = [{Tag, validate_opt(Name, Desc, undefined)} | Acc], 273 | validate_opts(Rest, Props, NewAcc) 274 | end. 275 | 276 | 277 | validate_opt(_Name, [], Value) -> 278 | Value; 279 | validate_opt(Name, Desc0, undefined) -> 280 | case lists:keytake(optional, 1, Desc0) of 281 | {value, {optional, true}, Desc1} -> 282 | {value, {default, Value}, Desc2} = lists:keytake(default, 1, Desc1), 283 | false = (Value == undefined), 284 | validate_opt(Name, Desc2, Value); 285 | _ -> 286 | ?MANGO_ERROR({missing_required_key, Name}) 287 | end; 288 | validate_opt(Name, [{tag, _} | Rest], Value) -> 289 | % Tags aren't really validated 290 | validate_opt(Name, Rest, Value); 291 | validate_opt(Name, [{optional, _} | Rest], Value) -> 292 | % A value was specified for an optional value 293 | validate_opt(Name, Rest, Value); 294 | validate_opt(Name, [{default, _} | Rest], Value) -> 295 | % A value was specified for an optional value 296 | validate_opt(Name, Rest, Value); 297 | validate_opt(Name, [{assert, Value} | Rest], Value) -> 298 | validate_opt(Name, Rest, Value); 299 | validate_opt(Name, [{assert, Expect} | _], Found) -> 300 | ?MANGO_ERROR({invalid_value, Name, Expect, Found}); 301 | validate_opt(Name, [{validator, Fun} | Rest], Value) -> 302 | case Fun(Value) of 303 | {ok, Validated} -> 304 | validate_opt(Name, Rest, Validated); 305 | false -> 306 | ?MANGO_ERROR({invalid_value, Name, Value}) 307 | end. 308 | 309 | 310 | -------------------------------------------------------------------------------- /src/mango_sort.erl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -module(mango_sort). 14 | 15 | -export([ 16 | new/1, 17 | to_json/1, 18 | fields/1, 19 | directions/1 20 | ]). 21 | 22 | 23 | -include("mango.hrl"). 24 | 25 | 26 | new(Fields) when is_list(Fields) -> 27 | Sort = {[sort_field(Field) || Field <- Fields]}, 28 | validate(Sort), 29 | {ok, Sort}; 30 | new(Else) -> 31 | ?MANGO_ERROR({invalid_sort_json, Else}). 32 | 33 | 34 | to_json({Fields}) -> 35 | to_json(Fields); 36 | to_json([]) -> 37 | []; 38 | to_json([{Name, Dir} | Rest]) -> 39 | [{[{Name, Dir}]} | to_json(Rest)]. 40 | 41 | 42 | fields({Props}) -> 43 | [Name || {Name, _Dir} <- Props]. 44 | 45 | 46 | directions({Props}) -> 47 | [Dir || {_Name, Dir} <- Props]. 48 | 49 | 50 | sort_field(Field) when is_binary(Field) -> 51 | {Field, <<"asc">>}; 52 | sort_field({[{Name, <<"asc">>}]}) when is_binary(Name) -> 53 | {Name, <<"asc">>}; 54 | sort_field({[{Name, <<"desc">>}]}) when is_binary(Name) -> 55 | {Name, <<"desc">>}; 56 | sort_field({Name, BadDir}) when is_binary(Name) -> 57 | ?MANGO_ERROR({invalid_sort_dir, BadDir}); 58 | sort_field(Else) -> 59 | ?MANGO_ERROR({invalid_sort_field, Else}). 60 | 61 | 62 | validate({Props}) -> 63 | % Assert each field is in the same direction 64 | % until we support mixed direction sorts. 65 | Dirs = [D || {_, D} <- Props], 66 | case lists:usort(Dirs) of 67 | [] -> 68 | ok; 69 | [_] -> 70 | ok; 71 | _ -> 72 | ?MANGO_ERROR({unsupported, mixed_sort}) 73 | end. 74 | -------------------------------------------------------------------------------- /src/mango_util.erl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -module(mango_util). 14 | 15 | 16 | -export([ 17 | open_doc/2, 18 | open_ddocs/1, 19 | load_ddoc/2, 20 | 21 | defer/3, 22 | do_defer/3, 23 | 24 | assert_ejson/1, 25 | 26 | to_lower/1, 27 | 28 | enc_dbname/1, 29 | dec_dbname/1, 30 | 31 | enc_hex/1, 32 | dec_hex/1, 33 | 34 | lucene_escape_field/1, 35 | lucene_escape_query_value/1, 36 | lucene_escape_user/1, 37 | is_number_string/1, 38 | 39 | check_lang/1, 40 | 41 | has_suffix/2, 42 | 43 | join/2, 44 | 45 | parse_field/1, 46 | 47 | cached_re/2 48 | ]). 49 | 50 | 51 | -include_lib("couch/include/couch_db.hrl"). 52 | -include("mango.hrl"). 53 | 54 | -define(DIGITS, "(\\p{N}+)"). 55 | -define(HEXDIGITS, "([0-9a-fA-F]+)"). 56 | -define(EXP, "[eE][+-]?" ++ ?DIGITS). 57 | -define(NUMSTRING, 58 | "[\\x00-\\x20]*" ++ "[+-]?(" ++ "NaN|" 59 | ++ "Infinity|" ++ "(((" 60 | ++ ?DIGITS 61 | ++ "(\\.)?(" 62 | ++ ?DIGITS 63 | ++ "?)(" 64 | ++ ?EXP 65 | ++ ")?)|" 66 | ++ "(\\.(" 67 | ++ ?DIGITS 68 | ++ ")(" 69 | ++ ?EXP 70 | ++ ")?)|" 71 | ++ "((" 72 | ++ "(0[xX]" 73 | ++ ?HEXDIGITS 74 | ++ "(\\.)?)|" 75 | ++ "(0[xX]" 76 | ++ ?HEXDIGITS 77 | ++ "?(\\.)" 78 | ++ ?HEXDIGITS 79 | ++ ")" 80 | ++ ")[pP][+-]?" ++ ?DIGITS ++ "))" ++ "[fFdD]?))" ++ "[\\x00-\\x20]*"). 81 | 82 | 83 | open_doc(Db, DocId) -> 84 | Opts = [deleted], 85 | case mango_util:defer(fabric, open_doc, [Db, DocId, Opts]) of 86 | {ok, Doc} -> 87 | {ok, Doc}; 88 | {not_found, _} -> 89 | not_found; 90 | _ -> 91 | ?MANGO_ERROR({error_loading_doc, DocId}) 92 | end. 93 | 94 | 95 | open_ddocs(Db) -> 96 | case mango_util:defer(fabric, design_docs, [Db]) of 97 | {ok, Docs} -> 98 | {ok, Docs}; 99 | _ -> 100 | ?MANGO_ERROR(error_loading_ddocs) 101 | end. 102 | 103 | 104 | load_ddoc(Db, DDocId) -> 105 | case mango_util:open_doc(Db, DDocId) of 106 | {ok, Doc} -> 107 | {ok, check_lang(Doc)}; 108 | not_found -> 109 | Body = {[ 110 | {<<"language">>, <<"query">>} 111 | ]}, 112 | {ok, #doc{id = DDocId, body = Body}} 113 | end. 114 | 115 | 116 | defer(Mod, Fun, Args) -> 117 | %twig:log(error, "MFA: ~p", [{Mod, Fun, Args}]), 118 | {Pid, Ref} = erlang:spawn_monitor(?MODULE, do_defer, [Mod, Fun, Args]), 119 | receive 120 | {'DOWN', Ref, process, Pid, {mango_defer_ok, Value}} -> 121 | Value; 122 | {'DOWN', Ref, process, Pid, {mango_defer_throw, Value}} -> 123 | erlang:throw(Value); 124 | {'DOWN', Ref, process, Pid, {mango_defer_error, Value}} -> 125 | erlang:error(Value); 126 | {'DOWN', Ref, process, Pid, {mango_defer_exit, Value}} -> 127 | erlang:exit(Value) 128 | end. 129 | 130 | 131 | do_defer(Mod, Fun, Args) -> 132 | try erlang:apply(Mod, Fun, Args) of 133 | Resp -> 134 | erlang:exit({mango_defer_ok, Resp}) 135 | catch 136 | throw:Error -> 137 | Stack = erlang:get_stacktrace(), 138 | twig:log(err, "Defered error: ~w~n ~p", [{throw, Error}, Stack]), 139 | erlang:exit({mango_defer_throw, Error}); 140 | error:Error -> 141 | Stack = erlang:get_stacktrace(), 142 | twig:log(err, "Defered error: ~w~n ~p", [{error, Error}, Stack]), 143 | erlang:exit({mango_defer_error, Error}); 144 | exit:Error -> 145 | Stack = erlang:get_stacktrace(), 146 | twig:log(err, "Defered error: ~w~n ~p", [{exit, Error}, Stack]), 147 | erlang:exit({mango_defer_exit, Error}) 148 | end. 149 | 150 | 151 | assert_ejson({Props}) -> 152 | assert_ejson_obj(Props); 153 | assert_ejson(Vals) when is_list(Vals) -> 154 | assert_ejson_arr(Vals); 155 | assert_ejson(null) -> 156 | true; 157 | assert_ejson(true) -> 158 | true; 159 | assert_ejson(false) -> 160 | true; 161 | assert_ejson(String) when is_binary(String) -> 162 | true; 163 | assert_ejson(Number) when is_number(Number) -> 164 | true; 165 | assert_ejson(_Else) -> 166 | false. 167 | 168 | 169 | assert_ejson_obj([]) -> 170 | true; 171 | assert_ejson_obj([{Key, Val} | Rest]) when is_binary(Key) -> 172 | case assert_ejson(Val) of 173 | true -> 174 | assert_ejson_obj(Rest); 175 | false -> 176 | false 177 | end; 178 | assert_ejson_obj(_Else) -> 179 | false. 180 | 181 | 182 | assert_ejson_arr([]) -> 183 | true; 184 | assert_ejson_arr([Val | Rest]) -> 185 | case assert_ejson(Val) of 186 | true -> 187 | assert_ejson_arr(Rest); 188 | false -> 189 | false 190 | end. 191 | 192 | 193 | check_lang(#doc{id = Id, deleted = true}) -> 194 | Body = {[ 195 | {<<"language">>, <<"query">>} 196 | ]}, 197 | #doc{id = Id, body = Body}; 198 | check_lang(#doc{body = {Props}} = Doc) -> 199 | case lists:keyfind(<<"language">>, 1, Props) of 200 | {<<"language">>, <<"query">>} -> 201 | Doc; 202 | Else -> 203 | ?MANGO_ERROR({invalid_ddoc_lang, Else}) 204 | end. 205 | 206 | 207 | to_lower(Key) when is_binary(Key) -> 208 | KStr = binary_to_list(Key), 209 | KLower = string:to_lower(KStr), 210 | list_to_binary(KLower). 211 | 212 | 213 | enc_dbname(<<>>) -> 214 | <<>>; 215 | enc_dbname(<>) -> 216 | Bytes = enc_db_byte(A), 217 | Tail = enc_dbname(Rest), 218 | <>. 219 | 220 | 221 | enc_db_byte(N) when N >= $a, N =< $z -> <>; 222 | enc_db_byte(N) when N >= $0, N =< $9 -> <>; 223 | enc_db_byte(N) when N == $/; N == $_; N == $- -> <>; 224 | enc_db_byte(N) -> 225 | H = enc_hex_byte(N div 16), 226 | L = enc_hex_byte(N rem 16), 227 | <<$$, H:8/integer, L:8/integer>>. 228 | 229 | 230 | dec_dbname(<<>>) -> 231 | <<>>; 232 | dec_dbname(<<$$, _:8/integer>>) -> 233 | throw(invalid_dbname_encoding); 234 | dec_dbname(<<$$, H:8/integer, L:8/integer, Rest/binary>>) -> 235 | Byte = (dec_hex_byte(H) bsl 4) bor dec_hex_byte(L), 236 | Tail = dec_dbname(Rest), 237 | <>; 238 | dec_dbname(<>) -> 239 | Tail = dec_dbname(Rest), 240 | <>. 241 | 242 | 243 | enc_hex(<<>>) -> 244 | <<>>; 245 | enc_hex(<>) -> 246 | H = enc_hex_byte(V div 16), 247 | L = enc_hex_byte(V rem 16), 248 | Tail = enc_hex(Rest), 249 | <>. 250 | 251 | 252 | enc_hex_byte(N) when N >= 0, N < 10 -> $0 + N; 253 | enc_hex_byte(N) when N >= 10, N < 16 -> $a + (N - 10); 254 | enc_hex_byte(N) -> throw({invalid_hex_value, N}). 255 | 256 | 257 | dec_hex(<<>>) -> 258 | <<>>; 259 | dec_hex(<<_:8/integer>>) -> 260 | throw(invalid_hex_string); 261 | dec_hex(<>) -> 262 | Byte = (dec_hex_byte(H) bsl 4) bor dec_hex_byte(L), 263 | Tail = dec_hex(Rest), 264 | <>. 265 | 266 | 267 | dec_hex_byte(N) when N >= $0, N =< $9 -> (N - $0); 268 | dec_hex_byte(N) when N >= $a, N =< $f -> (N - $a) + 10; 269 | dec_hex_byte(N) when N >= $A, N =< $F -> (N - $A) + 10; 270 | dec_hex_byte(N) -> throw({invalid_hex_character, N}). 271 | 272 | 273 | 274 | lucene_escape_field(Bin) when is_binary(Bin) -> 275 | Str = binary_to_list(Bin), 276 | Enc = lucene_escape_field(Str), 277 | iolist_to_binary(Enc); 278 | lucene_escape_field([H | T]) when is_number(H), H >= 0, H =< 255 -> 279 | if 280 | H >= $a, $z >= H -> 281 | [H | lucene_escape_field(T)]; 282 | H >= $A, $Z >= H -> 283 | [H | lucene_escape_field(T)]; 284 | H >= $0, $9 >= H -> 285 | [H | lucene_escape_field(T)]; 286 | true -> 287 | Hi = enc_hex_byte(H div 16), 288 | Lo = enc_hex_byte(H rem 16), 289 | [$_, Hi, Lo | lucene_escape_field(T)] 290 | end; 291 | lucene_escape_field([]) -> 292 | []. 293 | 294 | 295 | lucene_escape_query_value(IoList) when is_list(IoList) -> 296 | lucene_escape_query_value(iolist_to_binary(IoList)); 297 | lucene_escape_query_value(Bin) when is_binary(Bin) -> 298 | IoList = lucene_escape_qv(Bin), 299 | iolist_to_binary(IoList). 300 | 301 | 302 | % This escapes the special Lucene query characters 303 | % listed below as well as any whitespace. 304 | % 305 | % + - && || ! ( ) { } [ ] ^ ~ * ? : \ " / 306 | % 307 | 308 | lucene_escape_qv(<<>>) -> []; 309 | lucene_escape_qv(<<"&&", Rest/binary>>) -> 310 | ["\\&&" | lucene_escape_qv(Rest)]; 311 | lucene_escape_qv(<<"||", Rest/binary>>) -> 312 | ["\\||" | lucene_escape_qv(Rest)]; 313 | lucene_escape_qv(<>) -> 314 | NeedsEscape = "+-(){}[]!^~*?:/\\\" \t\r\n", 315 | Out = case lists:member(C, NeedsEscape) of 316 | true -> ["\\", C]; 317 | false -> [C] 318 | end, 319 | Out ++ lucene_escape_qv(Rest). 320 | 321 | 322 | lucene_escape_user(Field) -> 323 | {ok, Path} = parse_field(Field), 324 | Escaped = [mango_util:lucene_escape_field(P) || P <- Path], 325 | iolist_to_binary(join(".", Escaped)). 326 | 327 | 328 | has_suffix(Bin, Suffix) when is_binary(Bin), is_binary(Suffix) -> 329 | SBin = size(Bin), 330 | SSuffix = size(Suffix), 331 | if SBin < SSuffix -> false; true -> 332 | PSize = SBin - SSuffix, 333 | case Bin of 334 | <<_:PSize/binary, Suffix/binary>> -> 335 | true; 336 | _ -> 337 | false 338 | end 339 | end. 340 | 341 | 342 | join(_Sep, [Item]) -> 343 | [Item]; 344 | join(Sep, [Item | Rest]) -> 345 | [Item, Sep | join(Sep, Rest)]. 346 | 347 | 348 | is_number_string(Value) when is_binary(Value) -> 349 | is_number_string(binary_to_list(Value)); 350 | is_number_string(Value) when is_list(Value)-> 351 | MP = cached_re(mango_numstring_re, ?NUMSTRING), 352 | case re:run(Value, MP) of 353 | nomatch -> 354 | false; 355 | _ -> 356 | true 357 | end. 358 | 359 | cached_re(Name, RE) -> 360 | case mochiglobal:get(Name) of 361 | undefined -> 362 | {ok, MP} = re:compile(RE), 363 | ok = mochiglobal:put(Name, MP), 364 | MP; 365 | MP -> 366 | MP 367 | end. 368 | 369 | parse_field(Field) -> 370 | case binary:match(Field, <<"\\">>, []) of 371 | nomatch -> 372 | % Fast path, no regex required 373 | {ok, check_non_empty(Field, binary:split(Field, <<".">>, [global]))}; 374 | _ -> 375 | parse_field_slow(Field) 376 | end. 377 | 378 | parse_field_slow(Field) -> 379 | Path = lists:map(fun 380 | (P) when P =:= <<>> -> 381 | ?MANGO_ERROR({invalid_field_name, Field}); 382 | (P) -> 383 | re:replace(P, <<"\\\\">>, <<>>, [global, {return, binary}]) 384 | end, re:split(Field, <<"(?>)), 385 | {ok, Path}. 386 | 387 | 388 | check_non_empty(Field, Parts) -> 389 | case lists:member(<<>>, Parts) of 390 | true -> 391 | ?MANGO_ERROR({invalid_field_name, Field}); 392 | false -> 393 | Parts 394 | end. 395 | 396 | 397 | -ifdef(TEST). 398 | -include_lib("eunit/include/eunit.hrl"). 399 | 400 | parse_field_test() -> 401 | ?assertEqual({ok, [<<"ab">>]}, parse_field(<<"ab">>)), 402 | ?assertEqual({ok, [<<"a">>, <<"b">>]}, parse_field(<<"a.b">>)), 403 | ?assertEqual({ok, [<<"a.b">>]}, parse_field(<<"a\\.b">>)), 404 | ?assertEqual({ok, [<<"a">>, <<"b">>, <<"c">>]}, parse_field(<<"a.b.c">>)), 405 | ?assertEqual({ok, [<<"a">>, <<"b.c">>]}, parse_field(<<"a.b\\.c">>)), 406 | Exception = {mango_error, ?MODULE, {invalid_field_name, <<"a..b">>}}, 407 | ?assertThrow(Exception, parse_field(<<"a..b">>)). 408 | 409 | is_number_string_test() -> 410 | ?assert(is_number_string("0")), 411 | ?assert(is_number_string("1")), 412 | ?assert(is_number_string("1.0")), 413 | ?assert(is_number_string("1.0E10")), 414 | ?assert(is_number_string("0d")), 415 | ?assert(is_number_string("-1")), 416 | ?assert(is_number_string("-1.0")), 417 | ?assertNot(is_number_string("hello")), 418 | ?assertNot(is_number_string("")), 419 | ?assertMatch({match, _}, re:run("1.0", mochiglobal:get(mango_numstring_re))). 420 | 421 | -endif. 422 | -------------------------------------------------------------------------------- /test/01-index-crud-test.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | # use this file except in compliance with the License. You may obtain a copy of 3 | # the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | import random 14 | 15 | import mango 16 | 17 | 18 | class IndexCrudTests(mango.DbPerClass): 19 | def test_bad_fields(self): 20 | bad_fields = [ 21 | None, 22 | True, 23 | False, 24 | "bing", 25 | 2.0, 26 | {"foo": "bar"}, 27 | [{"foo": 2}], 28 | [{"foo": "asc", "bar": "desc"}], 29 | [{"foo": "asc"}, {"bar": "desc"}] 30 | ] 31 | for fields in bad_fields: 32 | try: 33 | self.db.create_index(fields) 34 | except Exception, e: 35 | assert e.response.status_code == 400 36 | else: 37 | raise AssertionError("bad create index") 38 | 39 | def test_bad_types(self): 40 | bad_types = [ 41 | None, 42 | True, 43 | False, 44 | 1.5, 45 | "foo", # Future support 46 | "geo", # Future support 47 | {"foo": "bar"}, 48 | ["baz", 3.0] 49 | ] 50 | for bt in bad_types: 51 | try: 52 | self.db.create_index(["foo"], idx_type=bt) 53 | except Exception, e: 54 | assert e.response.status_code == 400, (bt, e.response.status_code) 55 | else: 56 | raise AssertionError("bad create index") 57 | 58 | def test_bad_names(self): 59 | bad_names = [ 60 | True, 61 | False, 62 | 1.5, 63 | {"foo": "bar"}, 64 | [None, False] 65 | ] 66 | for bn in bad_names: 67 | try: 68 | self.db.create_index(["foo"], name=bn) 69 | except Exception, e: 70 | assert e.response.status_code == 400 71 | else: 72 | raise AssertionError("bad create index") 73 | try: 74 | self.db.create_index(["foo"], ddoc=bn) 75 | except Exception, e: 76 | assert e.response.status_code == 400 77 | else: 78 | raise AssertionError("bad create index") 79 | 80 | def test_create_idx_01(self): 81 | fields = ["foo", "bar"] 82 | ret = self.db.create_index(fields, name="idx_01") 83 | assert ret is True 84 | for idx in self.db.list_indexes(): 85 | if idx["name"] != "idx_01": 86 | continue 87 | assert idx["def"]["fields"] == [{"foo": "asc"}, {"bar": "asc"}] 88 | return 89 | raise AssertionError("index not created") 90 | 91 | def test_create_idx_01_exists(self): 92 | fields = ["foo", "bar"] 93 | ret = self.db.create_index(fields, name="idx_01") 94 | assert ret is False 95 | 96 | def test_create_idx_02(self): 97 | fields = ["baz", "foo"] 98 | ret = self.db.create_index(fields, name="idx_02") 99 | assert ret is True 100 | for idx in self.db.list_indexes(): 101 | if idx["name"] != "idx_02": 102 | continue 103 | assert idx["def"]["fields"] == [{"baz": "asc"}, {"foo": "asc"}] 104 | return 105 | raise AssertionError("index not created") 106 | 107 | def test_read_idx_doc(self): 108 | for idx in self.db.list_indexes(): 109 | if idx["type"] == "special": 110 | continue 111 | ddocid = idx["ddoc"] 112 | doc = self.db.open_doc(ddocid) 113 | assert doc["_id"] == ddocid 114 | info = self.db.ddoc_info(ddocid) 115 | assert info["name"] == ddocid 116 | 117 | def test_delete_idx_escaped(self): 118 | pre_indexes = self.db.list_indexes() 119 | ret = self.db.create_index(["bing"], name="idx_del_1") 120 | assert ret is True 121 | for idx in self.db.list_indexes(): 122 | if idx["name"] != "idx_del_1": 123 | continue 124 | assert idx["def"]["fields"] == [{"bing": "asc"}] 125 | self.db.delete_index(idx["ddoc"].replace("/", "%2F"), idx["name"]) 126 | post_indexes = self.db.list_indexes() 127 | assert pre_indexes == post_indexes 128 | 129 | def test_delete_idx_unescaped(self): 130 | pre_indexes = self.db.list_indexes() 131 | ret = self.db.create_index(["bing"], name="idx_del_2") 132 | assert ret is True 133 | for idx in self.db.list_indexes(): 134 | if idx["name"] != "idx_del_2": 135 | continue 136 | assert idx["def"]["fields"] == [{"bing": "asc"}] 137 | self.db.delete_index(idx["ddoc"], idx["name"]) 138 | post_indexes = self.db.list_indexes() 139 | assert pre_indexes == post_indexes 140 | 141 | def test_delete_idx_no_design(self): 142 | pre_indexes = self.db.list_indexes() 143 | ret = self.db.create_index(["bing"], name="idx_del_3") 144 | assert ret is True 145 | for idx in self.db.list_indexes(): 146 | if idx["name"] != "idx_del_3": 147 | continue 148 | assert idx["def"]["fields"] == [{"bing": "asc"}] 149 | self.db.delete_index(idx["ddoc"].split("/")[-1], idx["name"]) 150 | post_indexes = self.db.list_indexes() 151 | assert pre_indexes == post_indexes 152 | 153 | def test_bulk_delete(self): 154 | fields = ["field1"] 155 | ret = self.db.create_index(fields, name="idx_01") 156 | assert ret is True 157 | 158 | fields = ["field2"] 159 | ret = self.db.create_index(fields, name="idx_02") 160 | assert ret is True 161 | 162 | fields = ["field3"] 163 | ret = self.db.create_index(fields, name="idx_03") 164 | assert ret is True 165 | 166 | docids = [] 167 | 168 | for idx in self.db.list_indexes(): 169 | if idx["ddoc"] is not None: 170 | docids.append(idx["ddoc"]) 171 | 172 | docids.append("_design/this_is_not_an_index_name") 173 | 174 | ret = self.db.bulk_delete(docids) 175 | 176 | assert ret["error"][0]["id"] == "_design/this_is_not_an_index_name" 177 | assert len(ret["success"]) == 3 178 | 179 | for idx in self.db.list_indexes(): 180 | assert idx["type"] != "json" 181 | assert idx["type"] != "text" 182 | 183 | def test_recreate_index(self): 184 | pre_indexes = self.db.list_indexes() 185 | for i in range(5): 186 | ret = self.db.create_index(["bing"], name="idx_recreate") 187 | assert ret is True 188 | for idx in self.db.list_indexes(): 189 | if idx["name"] != "idx_recreate": 190 | continue 191 | assert idx["def"]["fields"] == [{"bing": "asc"}] 192 | self.db.delete_index(idx["ddoc"], idx["name"]) 193 | break 194 | post_indexes = self.db.list_indexes() 195 | assert pre_indexes == post_indexes 196 | 197 | def test_delete_misisng(self): 198 | # Missing design doc 199 | try: 200 | self.db.delete_index("this_is_not_a_design_doc_id", "foo") 201 | except Exception, e: 202 | assert e.response.status_code == 404 203 | else: 204 | raise AssertionError("bad index delete") 205 | 206 | # Missing view name 207 | indexes = self.db.list_indexes() 208 | not_special = [idx for idx in indexes if idx["type"] != "special"] 209 | idx = random.choice(not_special) 210 | ddocid = idx["ddoc"].split("/")[-1] 211 | try: 212 | self.db.delete_index(ddocid, "this_is_not_an_index_name") 213 | except Exception, e: 214 | assert e.response.status_code == 404 215 | else: 216 | raise AssertionError("bad index delete") 217 | 218 | # Bad view type 219 | try: 220 | self.db.delete_index(ddocid, idx["name"], idx_type="not_a_real_type") 221 | except Exception, e: 222 | assert e.response.status_code == 404 223 | else: 224 | raise AssertionError("bad index delete") 225 | 226 | def test_create_text_idx(self): 227 | fields = [ 228 | {"name":"stringidx", "type" : "string"}, 229 | {"name":"booleanidx", "type": "boolean"} 230 | ] 231 | ret = self.db.create_text_index(fields=fields, name="text_idx_01") 232 | assert ret is True 233 | for idx in self.db.list_indexes(): 234 | if idx["name"] != "text_idx_01": 235 | continue 236 | print idx["def"] 237 | assert idx["def"]["fields"] == [ 238 | {"stringidx": "string"}, 239 | {"booleanidx": "boolean"} 240 | ] 241 | return 242 | raise AssertionError("index not created") 243 | 244 | def test_create_bad_text_idx(self): 245 | bad_fields = [ 246 | True, 247 | False, 248 | "bing", 249 | 2.0, 250 | ["foo", "bar"], 251 | [{"name": "foo2"}], 252 | [{"name": "foo3", "type": "garbage"}], 253 | [{"type": "number"}], 254 | [{"name": "age", "type": "number"} , {"name": "bad"}], 255 | [{"name": "age", "type": "number"} , "bla"] 256 | ] 257 | for fields in bad_fields: 258 | try: 259 | self.db.create_text_index(fields=fields) 260 | except Exception, e: 261 | assert e.response.status_code == 400 262 | else: 263 | raise AssertionError("bad create text index") 264 | -------------------------------------------------------------------------------- /test/02-basic-find-test.py: -------------------------------------------------------------------------------- 1 | # -*- coding: latin-1 -*- 2 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 3 | # use this file except in compliance with the License. You may obtain a copy of 4 | # the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 10 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 11 | # License for the specific language governing permissions and limitations under 12 | # the License. 13 | 14 | 15 | import mango 16 | 17 | 18 | class BasicFindTests(mango.UserDocsTests): 19 | 20 | def test_bad_selector(self): 21 | bad_selectors = [ 22 | None, 23 | True, 24 | False, 25 | 1.0, 26 | "foobarbaz", 27 | {"foo":{"$not_an_op": 2}}, 28 | {"$gt":2}, 29 | [None, "bing"] 30 | ] 31 | for bs in bad_selectors: 32 | try: 33 | self.db.find(bs) 34 | except Exception, e: 35 | assert e.response.status_code == 400 36 | else: 37 | raise AssertionError("bad find") 38 | 39 | def test_bad_limit(self): 40 | bad_limits = [ 41 | None, 42 | True, 43 | False, 44 | -1, 45 | 1.2, 46 | "no limit!", 47 | {"foo": "bar"}, 48 | [2] 49 | ], 50 | for bl in bad_limits: 51 | try: 52 | self.db.find({"int":{"$gt":2}}, limit=bl) 53 | except Exception, e: 54 | assert e.response.status_code == 400 55 | else: 56 | raise AssertionError("bad find") 57 | 58 | def test_bad_skip(self): 59 | bad_skips = [ 60 | None, 61 | True, 62 | False, 63 | -3, 64 | 1.2, 65 | "no limit!", 66 | {"foo": "bar"}, 67 | [2] 68 | ], 69 | for bs in bad_skips: 70 | try: 71 | self.db.find({"int":{"$gt":2}}, skip=bs) 72 | except Exception, e: 73 | assert e.response.status_code == 400 74 | else: 75 | raise AssertionError("bad find") 76 | 77 | def test_bad_sort(self): 78 | bad_sorts = [ 79 | None, 80 | True, 81 | False, 82 | 1.2, 83 | "no limit!", 84 | {"foo": "bar"}, 85 | [2], 86 | [{"foo":"asc", "bar": "asc"}], 87 | [{"foo":"asc"}, {"bar":"desc"}], 88 | ], 89 | for bs in bad_sorts: 90 | try: 91 | self.db.find({"int":{"$gt":2}}, sort=bs) 92 | except Exception, e: 93 | assert e.response.status_code == 400 94 | else: 95 | raise AssertionError("bad find") 96 | 97 | def test_bad_fields(self): 98 | bad_fields = [ 99 | None, 100 | True, 101 | False, 102 | 1.2, 103 | "no limit!", 104 | {"foo": "bar"}, 105 | [2], 106 | [[]], 107 | ["foo", 2.0], 108 | ], 109 | for bf in bad_fields: 110 | try: 111 | self.db.find({"int":{"$gt":2}}, fields=bf) 112 | except Exception, e: 113 | assert e.response.status_code == 400 114 | else: 115 | raise AssertionError("bad find") 116 | 117 | def test_bad_r(self): 118 | bad_rs = [ 119 | None, 120 | True, 121 | False, 122 | 1.2, 123 | "no limit!", 124 | {"foo": "bar"}, 125 | [2], 126 | ], 127 | for br in bad_rs: 128 | try: 129 | self.db.find({"int":{"$gt":2}}, r=br) 130 | except Exception, e: 131 | assert e.response.status_code == 400 132 | else: 133 | raise AssertionError("bad find") 134 | 135 | def test_bad_conflicts(self): 136 | bad_conflicts = [ 137 | None, 138 | 1.2, 139 | "no limit!", 140 | {"foo": "bar"}, 141 | [2], 142 | ], 143 | for bc in bad_conflicts: 144 | try: 145 | self.db.find({"int":{"$gt":2}}, conflicts=bc) 146 | except Exception, e: 147 | assert e.response.status_code == 400 148 | else: 149 | raise AssertionError("bad find") 150 | 151 | def test_simple_find(self): 152 | docs = self.db.find({"age": {"$lt": 35}}) 153 | assert len(docs) == 3 154 | assert docs[0]["user_id"] == 9 155 | assert docs[1]["user_id"] == 1 156 | assert docs[2]["user_id"] == 7 157 | 158 | def test_multi_cond_and(self): 159 | docs = self.db.find({"manager": True, "location.city": "Longbranch"}) 160 | assert len(docs) == 1 161 | assert docs[0]["user_id"] == 7 162 | 163 | def test_multi_cond_or(self): 164 | docs = self.db.find({ 165 | "$and":[ 166 | {"age":{"$gte": 75}}, 167 | {"$or": [ 168 | {"name.first": "Mathis"}, 169 | {"name.first": "Whitley"} 170 | ]} 171 | ] 172 | }) 173 | assert len(docs) == 2 174 | assert docs[0]["user_id"] == 11 175 | assert docs[1]["user_id"] == 13 176 | 177 | def test_multi_col_idx(self): 178 | docs = self.db.find({ 179 | "location.state": {"$and": [ 180 | {"$gt": "Hawaii"}, 181 | {"$lt": "Maine"} 182 | ]}, 183 | "location.city": {"$lt": "Longbranch"} 184 | }) 185 | assert len(docs) == 1 186 | assert docs[0]["user_id"] == 6 187 | 188 | def test_missing_not_indexed(self): 189 | docs = self.db.find({"favorites.3": "C"}) 190 | assert len(docs) == 1 191 | assert docs[0]["user_id"] == 6 192 | 193 | docs = self.db.find({"favorites.3": None}) 194 | assert len(docs) == 0 195 | 196 | docs = self.db.find({"twitter": {"$gt": None}}) 197 | assert len(docs) == 4 198 | assert docs[0]["user_id"] == 1 199 | assert docs[1]["user_id"] == 4 200 | assert docs[2]["user_id"] == 0 201 | assert docs[3]["user_id"] == 13 202 | 203 | def test_limit(self): 204 | docs = self.db.find({"age": {"$gt": 0}}) 205 | assert len(docs) == 15 206 | for l in [0, 1, 5, 14]: 207 | docs = self.db.find({"age": {"$gt": 0}}, limit=l) 208 | assert len(docs) == l 209 | 210 | def test_skip(self): 211 | docs = self.db.find({"age": {"$gt": 0}}) 212 | assert len(docs) == 15 213 | for s in [0, 1, 5, 14]: 214 | docs = self.db.find({"age": {"$gt": 0}}, skip=s) 215 | assert len(docs) == (15 - s) 216 | 217 | def test_sort(self): 218 | docs1 = self.db.find({"age": {"$gt": 0}}, sort=[{"age":"asc"}]) 219 | docs2 = list(sorted(docs1, key=lambda d: d["age"])) 220 | assert docs1 is not docs2 and docs1 == docs2 221 | 222 | docs1 = self.db.find({"age": {"$gt": 0}}, sort=[{"age":"desc"}]) 223 | docs2 = list(reversed(sorted(docs1, key=lambda d: d["age"]))) 224 | assert docs1 is not docs2 and docs1 == docs2 225 | 226 | def test_fields(self): 227 | selector = {"age": {"$gt": 0}} 228 | docs = self.db.find(selector, fields=["user_id", "location.address"]) 229 | for d in docs: 230 | assert sorted(d.keys()) == ["location", "user_id"] 231 | assert sorted(d["location"].keys()) == ["address"] 232 | 233 | def test_r(self): 234 | for r in [1, 2, 3]: 235 | docs = self.db.find({"age": {"$gt": 0}}, r=r) 236 | assert len(docs) == 15 237 | 238 | def test_empty(self): 239 | try: 240 | self.db.find({}) 241 | except Exception, e: 242 | assert e.response.status_code == 400 243 | else: 244 | raise AssertionError("bad find") 245 | 246 | def test_empty_subsel(self): 247 | docs = self.db.find({ 248 | "_id": {"$gt": None}, 249 | "location": {} 250 | }) 251 | assert len(docs) == 0 252 | 253 | def test_empty_subsel_match(self): 254 | self.db.save_docs([{"user_id": "eo", "empty_obj": {}}]) 255 | docs = self.db.find({ 256 | "_id": {"$gt": None}, 257 | "empty_obj": {} 258 | }) 259 | assert len(docs) == 1 260 | assert docs[0]["user_id"] == "eo" 261 | 262 | def test_unsatisfiable_range(self): 263 | docs = self.db.find({ 264 | "$and":[ 265 | {"age":{"$gt": 0}}, 266 | {"age":{"$lt": 0}} 267 | ] 268 | }) 269 | assert len(docs) == 0 270 | -------------------------------------------------------------------------------- /test/03-operator-test.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | # use this file except in compliance with the License. You may obtain a copy of 3 | # the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | import mango 14 | 15 | 16 | class OperatorTests(mango.UserDocsTests): 17 | 18 | def test_all(self): 19 | docs = self.db.find({ 20 | "manager": True, 21 | "favorites": {"$all": ["Lisp", "Python"]} 22 | }) 23 | print docs 24 | assert len(docs) == 4 25 | assert docs[0]["user_id"] == 2 26 | assert docs[1]["user_id"] == 12 27 | assert docs[2]["user_id"] == 9 28 | assert docs[3]["user_id"] == 14 29 | 30 | def test_all_non_array(self): 31 | docs = self.db.find({ 32 | "manager": True, 33 | "location": {"$all": ["Ohai"]} 34 | }) 35 | assert len(docs) == 0 36 | 37 | def test_elem_match(self): 38 | emdocs = [ 39 | { 40 | "user_id": "a", 41 | "bang": [{ 42 | "foo": 1, 43 | "bar": 2 44 | }] 45 | }, 46 | { 47 | "user_id": "b", 48 | "bang": [{ 49 | "foo": 2, 50 | "bam": True 51 | }] 52 | } 53 | ] 54 | self.db.save_docs(emdocs, w=3) 55 | docs = self.db.find({ 56 | "_id": {"$gt": None}, 57 | "bang": {"$elemMatch": { 58 | "foo": {"$gte": 1}, 59 | "bam": True 60 | }} 61 | }) 62 | print docs 63 | assert len(docs) == 1 64 | assert docs[0]["user_id"] == "b" 65 | 66 | def test_in_operator_array(self): 67 | docs = self.db.find({ 68 | "manager": True, 69 | "favorites": {"$in": ["Ruby", "Python"]} 70 | }) 71 | assert len(docs) == 7 72 | assert docs[0]["user_id"] == 2 73 | assert docs[1]["user_id"] == 12 74 | 75 | def test_regex(self): 76 | docs = self.db.find({ 77 | "age": {"$gt": 40}, 78 | "location.state": {"$regex": "(?i)new.*"} 79 | }) 80 | assert len(docs) == 2 81 | assert docs[0]["user_id"] == 2 82 | assert docs[1]["user_id"] == 10 83 | 84 | def test_exists_false(self): 85 | docs = self.db.find({ 86 | "age": {"$gt": 0}, 87 | "twitter": {"$exists": False} 88 | }) 89 | user_ids = [2, 3, 5, 6, 7, 8, 9, 10, 11, 12, 14] 90 | assert len(docs) == len(user_ids) 91 | for doc in docs: 92 | assert doc["user_id"] in user_ids 93 | -------------------------------------------------------------------------------- /test/04-key-tests.py: -------------------------------------------------------------------------------- 1 | # -*- coding: latin-1 -*- 2 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 3 | # use this file except in compliance with the License. You may obtain a copy of 4 | # the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 10 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 11 | # License for the specific language governing permissions and limitations under 12 | # the License. 13 | 14 | 15 | import mango 16 | 17 | 18 | TEST_DOCS = [ 19 | { 20 | "type": "complex_key", 21 | "title": "normal key" 22 | }, 23 | { 24 | "type": "complex_key", 25 | "title": "key with dot", 26 | "dot.key": "dot's value", 27 | "none": { 28 | "dot": "none dot's value" 29 | }, 30 | "name.first" : "Kvothe" 31 | }, 32 | { 33 | "type": "complex_key", 34 | "title": "key with peso", 35 | "$key": "peso", 36 | "deep": { 37 | "$key": "deep peso" 38 | }, 39 | "name": {"first" : "Master Elodin"} 40 | }, 41 | { 42 | "type": "complex_key", 43 | "title": "unicode key", 44 | "": "apple" 45 | }, 46 | { 47 | "title": "internal_fields_format", 48 | "utf8-1[]:string" : "string", 49 | "utf8-2[]:boolean[]" : True, 50 | "utf8-3[]:number" : 9, 51 | "utf8-3[]:null" : None 52 | } 53 | ] 54 | 55 | 56 | class KeyTests(mango.DbPerClass): 57 | @classmethod 58 | def setUpClass(klass): 59 | super(KeyTests, klass).setUpClass() 60 | klass.db.save_docs(TEST_DOCS, w=3) 61 | klass.db.create_index(["type"], ddoc="view") 62 | klass.db.create_text_index(ddoc="text") 63 | 64 | def run_check(self, query, check, fields=None, indexes=None): 65 | if indexes is None: 66 | indexes = ["view", "text"] 67 | for idx in indexes: 68 | docs = self.db.find(query, fields=fields, use_index=idx) 69 | check(docs) 70 | 71 | def test_dot_key(self): 72 | query = {"type": "complex_key"} 73 | fields = ["title", "dot\\.key", "none.dot"] 74 | def check(docs): 75 | assert len(docs) == 4 76 | assert docs[1].has_key("dot.key") 77 | assert docs[1]["dot.key"] == "dot's value" 78 | assert docs[1].has_key("none") 79 | assert docs[1]["none"]["dot"] == "none dot's value" 80 | self.run_check(query, check, fields=fields) 81 | 82 | def test_peso_key(self): 83 | query = {"type": "complex_key"} 84 | fields = ["title", "$key", "deep.$key"] 85 | def check(docs): 86 | assert len(docs) == 4 87 | assert docs[2].has_key("$key") 88 | assert docs[2]["$key"] == "peso" 89 | assert docs[2].has_key("deep") 90 | assert docs[2]["deep"]["$key"] == "deep peso" 91 | self.run_check(query, check, fields=fields) 92 | 93 | def test_unicode_in_fieldname(self): 94 | query = {"type": "complex_key"} 95 | fields = ["title", ""] 96 | def check(docs): 97 | assert len(docs) == 4 98 | # note:  == \uf8ff 99 | assert docs[3].has_key(u'\uf8ff') 100 | assert docs[3][u'\uf8ff'] == "apple" 101 | self.run_check(query, check, fields=fields) 102 | 103 | # The rest of these tests are only run against the text 104 | # indexes because view indexes don't have to worry about 105 | # field *name* escaping in the index. 106 | 107 | def test_unicode_in_selector_field(self): 108 | query = {"" : "apple"} 109 | def check(docs): 110 | assert len(docs) == 1 111 | assert docs[0][u"\uf8ff"] == "apple" 112 | self.run_check(query, check, indexes=["text"]) 113 | 114 | def test_internal_field_tests(self): 115 | queries = [ 116 | {"utf8-1[]:string" : "string"}, 117 | {"utf8-2[]:boolean[]" : True}, 118 | {"utf8-3[]:number" : 9}, 119 | {"utf8-3[]:null" : None} 120 | ] 121 | def check(docs): 122 | assert len(docs) == 1 123 | assert docs[0]["title"] == "internal_fields_format" 124 | for query in queries: 125 | self.run_check(query, check, indexes=["text"]) 126 | 127 | def test_escape_period(self): 128 | query = {"name\\.first" : "Kvothe"} 129 | def check(docs): 130 | assert len(docs) == 1 131 | assert docs[0]["name.first"] == "Kvothe" 132 | self.run_check(query, check, indexes=["text"]) 133 | 134 | query = {"name.first" : "Kvothe"} 135 | def check_empty(docs): 136 | assert len(docs) == 0 137 | self.run_check(query, check_empty, indexes=["text"]) 138 | 139 | def test_object_period(self): 140 | query = {"name.first" : "Master Elodin"} 141 | def check(docs): 142 | assert len(docs) == 1 143 | assert docs[0]["title"] == "key with peso" 144 | self.run_check(query, check, indexes=["text"]) 145 | 146 | query = {"name\\.first" : "Master Elodin"} 147 | def check_empty(docs): 148 | assert len(docs) == 0 149 | self.run_check(query, check_empty, indexes=["text"]) 150 | -------------------------------------------------------------------------------- /test/05-index-selection-test.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | # use this file except in compliance with the License. You may obtain a copy of 3 | # the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | import mango 14 | import user_docs 15 | 16 | 17 | class IndexSelectionTests(mango.UserDocsTests): 18 | @classmethod 19 | def setUpClass(klass): 20 | super(IndexSelectionTests, klass).setUpClass() 21 | user_docs.add_text_indexes(klass.db, {}) 22 | 23 | def test_basic(self): 24 | resp = self.db.find({"name.last": "A last name"}, explain=True) 25 | assert resp["index"]["type"] == "json" 26 | 27 | def test_with_and(self): 28 | resp = self.db.find({ 29 | "name.first": "Stephanie", 30 | "name.last": "This doesn't have to match anything." 31 | }, explain=True) 32 | assert resp["index"]["type"] == "json" 33 | 34 | def test_with_text(self): 35 | resp = self.db.find({ 36 | "$text" : "Stephanie", 37 | "name.first": "Stephanie", 38 | "name.last": "This doesn't have to match anything." 39 | }, explain=True) 40 | assert resp["index"]["type"] == "text" 41 | 42 | def test_no_view_index(self): 43 | resp = self.db.find({"name.first": "Ohai!"}, explain=True) 44 | assert resp["index"]["type"] == "text" 45 | 46 | def test_with_or(self): 47 | resp = self.db.find({ 48 | "$or": [ 49 | {"name.first": "Stephanie"}, 50 | {"name.last": "This doesn't have to match anything."} 51 | ] 52 | }, explain=True) 53 | assert resp["index"]["type"] == "text" 54 | 55 | def test_use_most_columns(self): 56 | # ddoc id for the age index 57 | ddocid = "_design/ad3d537c03cd7c6a43cf8dff66ef70ea54c2b40f" 58 | resp = self.db.find({ 59 | "name.first": "Stephanie", 60 | "name.last": "Something or other", 61 | "age": {"$gt": 1} 62 | }, explain=True) 63 | assert resp["index"]["ddoc"] != "_design/" + ddocid 64 | 65 | resp = self.db.find({ 66 | "name.first": "Stephanie", 67 | "name.last": "Something or other", 68 | "age": {"$gt": 1} 69 | }, use_index=ddocid, explain=True) 70 | assert resp["index"]["ddoc"] == ddocid 71 | 72 | 73 | class MultiTextIndexSelectionTests(mango.UserDocsTests): 74 | @classmethod 75 | def setUpClass(klass): 76 | super(MultiTextIndexSelectionTests, klass).setUpClass() 77 | klass.db.create_text_index(ddoc="foo", analyzer="keyword") 78 | klass.db.create_text_index(ddoc="bar", analyzer="email") 79 | 80 | def test_view_ok_with_multi_text(self): 81 | resp = self.db.find({"name.last": "A last name"}, explain=True) 82 | assert resp["index"]["type"] == "json" 83 | 84 | def test_multi_text_index_is_error(self): 85 | try: 86 | self.db.find({"$text": "a query"}, explain=True) 87 | except Exception, e: 88 | assert e.response.status_code == 400 89 | 90 | def test_use_index_works(self): 91 | resp = self.db.find({"$text": "a query"}, use_index="foo", explain=True) 92 | assert resp["index"]["ddoc"] == "_design/foo" 93 | -------------------------------------------------------------------------------- /test/06-text-default-field-test.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | # use this file except in compliance with the License. You may obtain a copy of 3 | # the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | import mango 14 | 15 | 16 | 17 | class NoDefaultFieldTest(mango.UserDocsTextTests): 18 | 19 | DEFAULT_FIELD = False 20 | 21 | def test_basic(self): 22 | docs = self.db.find({"$text": "Ramona"}) 23 | # Or should this throw an error? 24 | assert len(docs) == 0 25 | 26 | def test_other_fields_exist(self): 27 | docs = self.db.find({"age": 22}) 28 | assert len(docs) == 1 29 | assert docs[0]["user_id"] == 9 30 | 31 | 32 | class NoDefaultFieldWithAnalyzer(mango.UserDocsTextTests): 33 | 34 | DEFAULT_FIELD = { 35 | "enabled": False, 36 | "analyzer": "keyword" 37 | } 38 | 39 | def test_basic(self): 40 | docs = self.db.find({"$text": "Ramona"}) 41 | assert len(docs) == 0 42 | 43 | def test_other_fields_exist(self): 44 | docs = self.db.find({"age": 22}) 45 | assert len(docs) == 1 46 | assert docs[0]["user_id"] == 9 47 | 48 | 49 | class DefaultFieldWithCustomAnalyzer(mango.UserDocsTextTests): 50 | 51 | DEFAULT_FIELD = { 52 | "enabled": True, 53 | "analyzer": "keyword" 54 | } 55 | 56 | def test_basic(self): 57 | docs = self.db.find({"$text": "Ramona"}) 58 | assert len(docs) == 1 59 | assert docs[0]["user_id"] == 9 60 | 61 | def test_not_analyzed(self): 62 | docs = self.db.find({"$text": "Lott Place"}) 63 | assert len(docs) == 1 64 | assert docs[0]["user_id"] == 9 65 | 66 | docs = self.db.find({"$text": "Lott"}) 67 | assert len(docs) == 0 68 | 69 | docs = self.db.find({"$text": "Place"}) 70 | assert len(docs) == 0 71 | -------------------------------------------------------------------------------- /test/07-text-custom-field-list-test.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | # use this file except in compliance with the License. You may obtain a copy of 3 | # the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | import mango 14 | 15 | 16 | 17 | class CustomFieldsTest(mango.UserDocsTextTests): 18 | 19 | FIELDS = [ 20 | {"name": "favorites.[]", "type": "string"}, 21 | {"name": "manager", "type": "boolean"}, 22 | {"name": "age", "type": "number"}, 23 | # These two are to test the default analyzer for 24 | # each field. 25 | {"name": "location.state", "type": "string"}, 26 | { 27 | "name": "location.address.street", 28 | "type": "string" 29 | }, 30 | {"name": "name\\.first", "type": "string"} 31 | ] 32 | 33 | def test_basic(self): 34 | docs = self.db.find({"age": 22}) 35 | assert len(docs) == 1 36 | assert docs[0]["user_id"] == 9 37 | 38 | def test_multi_field(self): 39 | docs = self.db.find({"age": 22, "manager": True}) 40 | assert len(docs) == 1 41 | assert docs[0]["user_id"] == 9 42 | 43 | docs = self.db.find({"age": 22, "manager": False}) 44 | assert len(docs) == 0 45 | 46 | def test_missing(self): 47 | self.db.find({"location.state": "Nevada"}) 48 | 49 | def test_missing_type(self): 50 | # Raises an exception 51 | try: 52 | self.db.find({"age": "foo"}) 53 | raise Exception("Should have thrown an HTTPError") 54 | except: 55 | return 56 | 57 | def test_field_analyzer_is_keyword(self): 58 | docs = self.db.find({"location.state": "New"}) 59 | assert len(docs) == 0 60 | 61 | docs = self.db.find({"location.state": "New Hampshire"}) 62 | assert len(docs) == 1 63 | assert docs[0]["user_id"] == 10 64 | 65 | # Since our FIELDS list only includes "name\\.first", we should 66 | # get an error when we try to search for "name.first", since the index 67 | # for that field does not exist. 68 | def test_escaped_field(self): 69 | docs = self.db.find({"name\\.first": "name dot first"}) 70 | assert len(docs) == 1 71 | assert docs[0]["name.first"] == "name dot first" 72 | 73 | try: 74 | self.db.find({"name.first": "name dot first"}) 75 | raise Exception("Should have thrown an HTTPError") 76 | except: 77 | return 78 | 79 | def test_filtered_search_fields(self): 80 | docs = self.db.find({"age": 22}, fields = ["age", "location.state"]) 81 | assert len(docs) == 1 82 | assert docs == [{"age": 22, "location": {"state": "Missouri"}}] 83 | 84 | docs = self.db.find({"age": 22}, fields = ["age", "Random Garbage"]) 85 | assert len(docs) == 1 86 | assert docs == [{"age": 22}] 87 | 88 | docs = self.db.find({"age": 22}, fields = ["favorites"]) 89 | assert len(docs) == 1 90 | assert docs == [{"favorites": ["Lisp", "Erlang", "Python"]}] 91 | 92 | docs = self.db.find({"age": 22}, fields = ["favorites.[]"]) 93 | assert len(docs) == 1 94 | assert docs == [{}] 95 | 96 | docs = self.db.find({"age": 22}, fields = ["all_fields"]) 97 | assert len(docs) == 1 98 | assert docs == [{}] 99 | -------------------------------------------------------------------------------- /test/08-text-limit-test.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | # use this file except in compliance with the License. You may obtain a copy of 3 | # the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | import mango 14 | import limit_docs 15 | 16 | class LimitTests(mango.LimitDocsTextTests): 17 | 18 | def test_limit_field(self): 19 | q = {"$or": [{"user_id" : {"$lt" : 10}}, {"filtered_array.[]": 1}]} 20 | docs = self.db.find(q, limit=10) 21 | assert len(docs) == 8 22 | for d in docs: 23 | assert d["user_id"] < 10 24 | 25 | def test_limit_field2(self): 26 | q = {"$or": [{"user_id" : {"$lt" : 20}}, {"filtered_array.[]": 1}]} 27 | docs = self.db.find(q, limit=10) 28 | assert len(docs) == 10 29 | for d in docs: 30 | assert d["user_id"] < 20 31 | 32 | def test_limit_field3(self): 33 | q = {"$or": [{"user_id" : {"$lt" : 100}}, {"filtered_array.[]": 1}]} 34 | docs = self.db.find(q, limit=1) 35 | assert len(docs) == 1 36 | for d in docs: 37 | assert d["user_id"] < 100 38 | 39 | def test_limit_field4(self): 40 | q = {"$or": [{"user_id" : {"$lt" : 0}}, {"filtered_array.[]": 1}]} 41 | docs = self.db.find(q, limit=35) 42 | assert len(docs) == 0 43 | 44 | def test_limit_field5(self): 45 | q = {"age": {"$exists": True}} 46 | docs = self.db.find(q, limit=250) 47 | print len(docs) 48 | assert len(docs) == 75 49 | for d in docs: 50 | assert d["age"] < 100 51 | 52 | def test_limit_skip_field1(self): 53 | q = {"$or": [{"user_id" : {"$lt" : 100}}, {"filtered_array.[]": 1}]} 54 | docs = self.db.find(q, limit=10, skip=20) 55 | assert len(docs) == 10 56 | for d in docs: 57 | assert d["user_id"] > 20 58 | 59 | def test_limit_skip_field2(self): 60 | q = {"$or": [{"user_id" : {"$lt" : 100}}, {"filtered_array.[]": 1}]} 61 | docs = self.db.find(q, limit=100, skip=100) 62 | assert len(docs) == 0 63 | 64 | def test_limit_skip_field3(self): 65 | q = {"$or": [{"user_id" : {"$lt" : 20}}, {"filtered_array.[]": 1}]} 66 | docs = self.db.find(q, limit=1, skip=30) 67 | assert len(docs) == 0 68 | 69 | def test_limit_skip_field4(self): 70 | q = {"$or": [{"user_id" : {"$lt" : 100}}, {"filtered_array.[]": 1}]} 71 | docs = self.db.find(q, limit=0, skip=0) 72 | assert len(docs) == 0 73 | 74 | def test_limit_skip_field5(self): 75 | q = {"$or": [{"user_id" : {"$lt" : 100}}, {"filtered_array.[]": 1}]} 76 | try: 77 | self.db.find(q, limit=-1) 78 | except Exception, e: 79 | assert e.response.status_code == 400 80 | else: 81 | raise AssertionError("Should have thrown error for negative limit") 82 | 83 | def test_limit_skip_field6(self): 84 | q = {"$or": [{"user_id" : {"$lt" : 100}}, {"filtered_array.[]": 1}]} 85 | try: 86 | self.db.find(q, skip=-1) 87 | except Exception, e: 88 | assert e.response.status_code == 400 89 | else: 90 | raise AssertionError("Should have thrown error for negative skip") 91 | 92 | # Basic test to ensure we can iterate through documents with a bookmark 93 | def test_limit_bookmark(self): 94 | for i in range(1, len(limit_docs.DOCS), 5): 95 | self.run_bookmark_check(i) 96 | 97 | for i in range(1, len(limit_docs.DOCS), 5): 98 | self.run_bookmark_sort_check(i) 99 | 100 | 101 | def run_bookmark_check(self, size): 102 | print size 103 | q = {"age": {"$gt": 0}} 104 | seen_docs = set() 105 | bm = None 106 | while True: 107 | json = self.db.find(q, limit=size, bookmark=bm, return_raw=True) 108 | for doc in json["docs"]: 109 | assert doc["_id"] not in seen_docs 110 | seen_docs.add(doc["_id"]) 111 | if not len(json["docs"]): 112 | break 113 | assert json["bookmark"] != bm 114 | bm = json["bookmark"] 115 | assert len(seen_docs) == len(limit_docs.DOCS) 116 | 117 | def run_bookmark_sort_check(self, size): 118 | q = {"age": {"$gt": 0}} 119 | seen_docs = set() 120 | bm = None 121 | age = 0 122 | while True: 123 | json = self.db.find(q, limit=size, bookmark=bm, sort=["age"], 124 | return_raw=True) 125 | for doc in json["docs"]: 126 | assert doc["_id"] not in seen_docs 127 | assert doc["age"] >= age 128 | age = doc["age"] 129 | seen_docs.add(doc["_id"]) 130 | if not len(json["docs"]): 131 | break 132 | assert json["bookmark"] != bm 133 | bm = json["bookmark"] 134 | assert len(seen_docs) == len(limit_docs.DOCS) 135 | -------------------------------------------------------------------------------- /test/09-text-sort-test.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | # use this file except in compliance with the License. You may obtain a copy of 3 | # the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | import mango 14 | import user_docs 15 | 16 | class SortTests(mango.UserDocsTextTests): 17 | 18 | def test_number_sort(self): 19 | q = {"age": {"$gt": 0}} 20 | docs = self.db.find(q, sort=["age:number"]) 21 | assert len(docs) == 15 22 | assert docs[0]["age"] == 22 23 | 24 | def test_number_sort_desc(self): 25 | q = {"age": {"$gt": 0}} 26 | docs = self.db.find(q, sort=[{"age": "desc"}]) 27 | assert len(docs) == 15 28 | assert docs[0]["age"] == 79 29 | 30 | q = {"manager": True} 31 | docs = self.db.find(q, sort=[{"age:number": "desc"}]) 32 | assert len(docs) == 11 33 | assert docs[0]["age"] == 79 34 | 35 | def test_string_sort(self): 36 | q = {"email": {"$gt": None}} 37 | docs = self.db.find(q, sort=["email:string"]) 38 | assert len(docs) == 15 39 | assert docs[0]["email"] == "abbottwatson@talkola.com" 40 | 41 | def test_notype_sort(self): 42 | q = {"email": {"$gt": None}} 43 | try: 44 | self.db.find(q, sort=["email"]) 45 | except Exception, e: 46 | assert e.response.status_code == 400 47 | else: 48 | raise AssertionError("Should have thrown error for sort") 49 | 50 | def test_array_sort(self): 51 | q = {"favorites": {"$exists": True}} 52 | docs = self.db.find(q, sort=["favorites.[]:string"]) 53 | assert len(docs) == 15 54 | assert docs[0]["user_id"] == 8 55 | 56 | def test_multi_sort(self): 57 | q = {"name": {"$exists": True}} 58 | docs = self.db.find(q, sort=["name.last:string", "age:number"]) 59 | assert len(docs) == 15 60 | assert docs[0]["name"] == {"last":"Ewing","first":"Shelly"} 61 | assert docs[1]["age"] == 22 62 | 63 | def test_guess_type_sort(self): 64 | q = {"$or": [{"age":{"$gt": 0}}, {"email": {"$gt": None}}]} 65 | docs = self.db.find(q, sort=["age"]) 66 | assert len(docs) == 15 67 | assert docs[0]["age"] == 22 68 | 69 | def test_guess_dup_type_sort(self): 70 | q = {"$and": [{"age":{"$gt": 0}}, {"email": {"$gt": None}}, 71 | {"age":{"$lte": 100}}]} 72 | docs = self.db.find(q, sort=["age"]) 73 | assert len(docs) == 15 74 | assert docs[0]["age"] == 22 75 | 76 | def test_ambiguous_type_sort(self): 77 | q = {"$or": [{"age":{"$gt": 0}}, {"email": {"$gt": None}}, 78 | {"age": "34"}]} 79 | try: 80 | self.db.find(q, sort=["age"]) 81 | except Exception, e: 82 | assert e.response.status_code == 400 83 | else: 84 | raise AssertionError("Should have thrown error for sort") 85 | 86 | def test_guess_multi_sort(self): 87 | q = {"$or": [{"age":{"$gt": 0}}, {"email": {"$gt": None}}, 88 | {"name.last": "Harvey"}]} 89 | docs = self.db.find(q, sort=["name.last", "age"]) 90 | assert len(docs) == 15 91 | assert docs[0]["name"] == {"last":"Ewing","first":"Shelly"} 92 | assert docs[1]["age"] == 22 93 | 94 | def test_guess_mix_sort(self): 95 | q = {"$or": [{"age":{"$gt": 0}}, {"email": {"$gt": None}}, 96 | {"name.last": "Harvey"}]} 97 | docs = self.db.find(q, sort=["name.last:string", "age"]) 98 | assert len(docs) == 15 99 | assert docs[0]["name"] == {"last":"Ewing","first":"Shelly"} 100 | assert docs[1]["age"] == 22 101 | -------------------------------------------------------------------------------- /test/10-disable-array-length-field-test.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | # use this file except in compliance with the License. You may obtain a copy of 3 | # the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | import mango 14 | 15 | 16 | class DisableIndexArrayLengthsTest(mango.UserDocsTextTests): 17 | 18 | @classmethod 19 | def setUpClass(klass): 20 | super(DisableIndexArrayLengthsTest, klass).setUpClass() 21 | klass.db.create_text_index(ddoc="disable_index_array_lengths", 22 | analyzer="keyword", index_array_lengths=False) 23 | klass.db.create_text_index(ddoc="explicit_enable_index_array_lengths", 24 | analyzer="keyword", index_array_lengths=True) 25 | 26 | def test_disable_index_array_length(self): 27 | docs = self.db.find({"favorites": {"$size": 4}}, 28 | use_index="disable_index_array_lengths") 29 | for d in docs: 30 | assert len(d["favorites"]) == 0 31 | 32 | def test_enable_index_array_length(self): 33 | docs = self.db.find({"favorites": {"$size": 4}}, 34 | use_index="explicit_enable_index_array_lengths") 35 | for d in docs: 36 | assert len(d["favorites"]) == 4 37 | -------------------------------------------------------------------------------- /test/README.md: -------------------------------------------------------------------------------- 1 | Mango Tests 2 | =========== 3 | 4 | To run these, do this in the top level directory: 5 | 6 | $ virtualenv venv 7 | $ source venv/bin/activate 8 | $ pip install nose requests 9 | $ pip install hypothesis 10 | $ nosetests 11 | 12 | -------------------------------------------------------------------------------- /test/limit_docs.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | # use this file except in compliance with the License. You may obtain a copy of 3 | # the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | import copy 14 | 15 | 16 | def setup(db, index_type="view"): 17 | db.recreate() 18 | db.save_docs(copy.deepcopy(DOCS)) 19 | if index_type == "view": 20 | add_view_indexes(db) 21 | elif index_type == "text": 22 | add_text_indexes(db) 23 | 24 | 25 | def add_text_indexes(db): 26 | db.create_text_index() 27 | 28 | 29 | DOCS = [ 30 | { 31 | "_id": "54af50626de419f5109c962f", 32 | "user_id": 0, 33 | "age": 10 34 | }, 35 | { 36 | "_id": "54af50622071121b25402dc3", 37 | "user_id": 1, 38 | "age": 11 39 | 40 | }, 41 | { 42 | "_id": "54af50623809e19159a3cdd0", 43 | "user_id": 2, 44 | "age": 12 45 | }, 46 | { 47 | "_id": "54af50629f45a0f49a441d01", 48 | "user_id": 3, 49 | "age": 13 50 | 51 | }, 52 | { 53 | "_id": "54af50620f1755c22359a362", 54 | "user_id": 4, 55 | "age": 14 56 | }, 57 | { 58 | "_id": "54af5062dd6f6c689ad2ca23", 59 | "user_id": 5, 60 | "age": 15 61 | }, 62 | { 63 | "_id": "54af50623e89b432be1187b8", 64 | "user_id": 6, 65 | "age": 16 66 | }, 67 | { 68 | "_id": "54af5062932a00270a3b5ab0", 69 | "user_id": 7, 70 | "age": 17 71 | 72 | }, 73 | { 74 | "_id": "54af5062df773d69174e3345", 75 | "filtered_array" : [1, 2, 3], 76 | "age": 18 77 | }, 78 | { 79 | "_id": "54af50629c1153b9e21e346d", 80 | "filtered_array" : [1, 2, 3], 81 | "age": 19 82 | }, 83 | { 84 | "_id": "54af5062dabb7cc4b60e0c95", 85 | "user_id": 10, 86 | "age": 20 87 | }, 88 | { 89 | "_id": "54af5062204996970a4439a2", 90 | "user_id": 11, 91 | "age": 21 92 | }, 93 | { 94 | "_id": "54af50629cea39e8ea52bfac", 95 | "user_id": 12, 96 | "age": 22 97 | }, 98 | { 99 | "_id": "54af50620597c094f75db2a1", 100 | "user_id": 13, 101 | "age": 23 102 | }, 103 | { 104 | "_id": "54af50628d4048de0010723c", 105 | "user_id": 14, 106 | "age": 24 107 | }, 108 | { 109 | "_id": "54af5062f339b6f44f52faf6", 110 | "user_id": 15, 111 | "age": 25 112 | }, 113 | { 114 | "_id": "54af5062a893f17ea4402031", 115 | "user_id": 16, 116 | "age": 26 117 | }, 118 | { 119 | "_id": "54af5062323dbc7077deb60a", 120 | "user_id": 17, 121 | "age": 27 122 | }, 123 | { 124 | "_id": "54af506224db85bd7fcd0243", 125 | "filtered_array" : [1, 2, 3], 126 | "age": 28 127 | }, 128 | { 129 | "_id": "54af506255bb551c9cc251bf", 130 | "filtered_array" : [1, 2, 3], 131 | "age": 29 132 | }, 133 | { 134 | "_id": "54af50625a97394e07d718a1", 135 | "filtered_array" : [1, 2, 3], 136 | "age": 30 137 | }, 138 | { 139 | "_id": "54af506223f51d586b4ef529", 140 | "user_id": 21, 141 | "age": 31 142 | }, 143 | { 144 | "_id": "54af50622740dede7d6117b7", 145 | "user_id": 22, 146 | "age": 32 147 | }, 148 | { 149 | "_id": "54af50624efc87684a52e8fb", 150 | "user_id": 23, 151 | "age": 33 152 | }, 153 | { 154 | "_id": "54af5062f40932760347799c", 155 | "user_id": 24, 156 | "age": 34 157 | }, 158 | { 159 | "_id": "54af5062d9f7361951ac645d", 160 | "user_id": 25, 161 | "age": 35 162 | }, 163 | { 164 | "_id": "54af5062f89aef302b37c3bc", 165 | "filtered_array" : [1, 2, 3], 166 | "age": 36 167 | }, 168 | { 169 | "_id": "54af5062498ec905dcb351f8", 170 | "filtered_array" : [1, 2, 3], 171 | "age": 37 172 | }, 173 | { 174 | "_id": "54af5062b1d2f2c5a85bdd7e", 175 | "user_id": 28, 176 | "age": 38 177 | }, 178 | { 179 | "_id": "54af50625061029c0dd942b5", 180 | "filtered_array" : [1, 2, 3], 181 | "age": 39 182 | }, 183 | { 184 | "_id": "54af50628b0d08a1d23c030a", 185 | "user_id": 30, 186 | "age": 40 187 | }, 188 | { 189 | "_id": "54af506271b6e3119eb31d46", 190 | "filtered_array" : [1, 2, 3], 191 | "age": 41 192 | }, 193 | { 194 | "_id": "54af5062b69f46424dfcf3e5", 195 | "user_id": 32, 196 | "age": 42 197 | }, 198 | { 199 | "_id": "54af5062ed00c7dbe4d1bdcf", 200 | "user_id": 33, 201 | "age": 43 202 | }, 203 | { 204 | "_id": "54af5062fb64e45180c9a90d", 205 | "user_id": 34, 206 | "age": 44 207 | }, 208 | { 209 | "_id": "54af5062241c72b067127b09", 210 | "user_id": 35, 211 | "age": 45 212 | }, 213 | { 214 | "_id": "54af50626a467d8b781a6d06", 215 | "user_id": 36, 216 | "age": 46 217 | }, 218 | { 219 | "_id": "54af50620e992d60af03bf86", 220 | "filtered_array" : [1, 2, 3], 221 | "age": 47 222 | }, 223 | { 224 | "_id": "54af506254f992aa3c51532f", 225 | "user_id": 38, 226 | "age": 48 227 | }, 228 | { 229 | "_id": "54af5062e99b20f301de39b9", 230 | "user_id": 39, 231 | "age": 49 232 | }, 233 | { 234 | "_id": "54af50624fbade6b11505b5d", 235 | "user_id": 40, 236 | "age": 50 237 | }, 238 | { 239 | "_id": "54af506278ad79b21e807ae4", 240 | "user_id": 41, 241 | "age": 51 242 | }, 243 | { 244 | "_id": "54af5062fc7a1dcb33f31d08", 245 | "user_id": 42, 246 | "age": 52 247 | }, 248 | { 249 | "_id": "54af5062ea2c954c650009cf", 250 | "user_id": 43, 251 | "age": 53 252 | }, 253 | { 254 | "_id": "54af506213576c2f09858266", 255 | "user_id": 44, 256 | "age": 54 257 | }, 258 | { 259 | "_id": "54af50624a05ac34c994b1c0", 260 | "user_id": 45, 261 | "age": 55 262 | }, 263 | { 264 | "_id": "54af50625a624983edf2087e", 265 | "user_id": 46, 266 | "age": 56 267 | }, 268 | { 269 | "_id": "54af50623de488c49d064355", 270 | "user_id": 47, 271 | "age": 57 272 | }, 273 | { 274 | "_id": "54af5062628b5df08661a9d5", 275 | "user_id": 48, 276 | "age": 58 277 | }, 278 | { 279 | "_id": "54af50620c706fc23032ae62", 280 | "user_id": 49, 281 | "age": 59 282 | }, 283 | { 284 | "_id": "54af5062509f1e2371fe1da4", 285 | "user_id": 50, 286 | "age": 60 287 | }, 288 | { 289 | "_id": "54af50625e96b22436791653", 290 | "user_id": 51, 291 | "age": 61 292 | }, 293 | { 294 | "_id": "54af5062a9cb71463bb9577f", 295 | "user_id": 52, 296 | "age": 62 297 | }, 298 | { 299 | "_id": "54af50624fea77a4221a4baf", 300 | "user_id": 53, 301 | "age": 63 302 | }, 303 | { 304 | "_id": "54af5062c63df0a147d2417e", 305 | "user_id": 54, 306 | "age": 64 307 | }, 308 | { 309 | "_id": "54af50623c56d78029316c9f", 310 | "user_id": 55, 311 | "age": 65 312 | }, 313 | { 314 | "_id": "54af5062167f6e13aa0dd014", 315 | "user_id": 56, 316 | "age": 66 317 | }, 318 | { 319 | "_id": "54af50621558abe77797d137", 320 | "filtered_array" : [1, 2, 3], 321 | "age": 67 322 | }, 323 | { 324 | "_id": "54af50624d5b36aa7cb5fa77", 325 | "user_id": 58, 326 | "age": 68 327 | }, 328 | { 329 | "_id": "54af50620d79118184ae66bd", 330 | "user_id": 59, 331 | "age": 69 332 | }, 333 | { 334 | "_id": "54af5062d18aafa5c4ca4935", 335 | "user_id": 60, 336 | "age": 71 337 | }, 338 | { 339 | "_id": "54af5062fd22a409649962f4", 340 | "filtered_array" : [1, 2, 3], 341 | "age": 72 342 | }, 343 | { 344 | "_id": "54af5062e31045a1908e89f9", 345 | "user_id": 62, 346 | "age": 73 347 | }, 348 | { 349 | "_id": "54af50624c062fcb4c59398b", 350 | "user_id": 63, 351 | "age": 74 352 | }, 353 | { 354 | "_id": "54af506241ec83430a15957f", 355 | "user_id": 64, 356 | "age": 75 357 | }, 358 | { 359 | "_id": "54af506224d0f888ae411101", 360 | "user_id": 65, 361 | "age": 76 362 | }, 363 | { 364 | "_id": "54af506272a971c6cf3ab6b8", 365 | "user_id": 66, 366 | "age": 77 367 | }, 368 | { 369 | "_id": "54af506221e25b485c95355b", 370 | "user_id": 67, 371 | "age": 78 372 | }, 373 | { 374 | "_id": "54af5062800f7f2ca73e9623", 375 | "user_id": 68, 376 | "age": 79 377 | }, 378 | { 379 | "_id": "54af5062bc962da30740534a", 380 | "user_id": 69, 381 | "age": 80 382 | }, 383 | { 384 | "_id": "54af50625102d6e210fc2efd", 385 | "filtered_array" : [1, 2, 3], 386 | "age": 81 387 | }, 388 | { 389 | "_id": "54af5062e014b9d039f02c5e", 390 | "user_id": 71, 391 | "age": 82 392 | }, 393 | { 394 | "_id": "54af5062fbd5e801dd217515", 395 | "user_id": 72, 396 | "age": 83 397 | }, 398 | { 399 | "_id": "54af50629971992b658fcb88", 400 | "user_id": 73, 401 | "age": 84 402 | }, 403 | { 404 | "_id": "54af5062607d53416c30bafd", 405 | "filtered_array" : [1, 2, 3], 406 | "age": 85 407 | } 408 | ] 409 | -------------------------------------------------------------------------------- /test/mango.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | # use this file except in compliance with the License. You may obtain a copy of 3 | # the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | import json 14 | import time 15 | import unittest 16 | import uuid 17 | 18 | import requests 19 | 20 | import friend_docs 21 | import user_docs 22 | import limit_docs 23 | 24 | 25 | def random_db_name(): 26 | return "mango_test_" + uuid.uuid4().hex 27 | 28 | 29 | class Database(object): 30 | def __init__(self, host, port, dbname, auth=None): 31 | self.host = host 32 | self.port = port 33 | self.dbname = dbname 34 | self.sess = requests.session() 35 | if auth is not None: 36 | self.sess.auth = auth 37 | self.sess.headers["Content-Type"] = "application/json" 38 | 39 | @property 40 | def url(self): 41 | return "http://{}:{}/{}".format(self.host, self.port, self.dbname) 42 | 43 | def path(self, parts): 44 | if isinstance(parts, (str, unicode)): 45 | parts = [parts] 46 | return "/".join([self.url] + parts) 47 | 48 | def create(self, q=1, n=3): 49 | r = self.sess.get(self.url) 50 | if r.status_code == 404: 51 | r = self.sess.put(self.url, params={"q":q, "n": n}) 52 | r.raise_for_status() 53 | 54 | def delete(self): 55 | r = self.sess.delete(self.url) 56 | 57 | def recreate(self): 58 | self.delete() 59 | time.sleep(1) 60 | self.create() 61 | time.sleep(1) 62 | 63 | def save_doc(self, doc): 64 | self.save_docs([doc]) 65 | 66 | def save_docs(self, docs, **kwargs): 67 | body = json.dumps({"docs": docs}) 68 | r = self.sess.post(self.path("_bulk_docs"), data=body, params=kwargs) 69 | r.raise_for_status() 70 | for doc, result in zip(docs, r.json()): 71 | doc["_id"] = result["id"] 72 | doc["_rev"] = result["rev"] 73 | 74 | def open_doc(self, docid): 75 | r = self.sess.get(self.path(docid)) 76 | r.raise_for_status() 77 | return r.json() 78 | 79 | def ddoc_info(self, ddocid): 80 | r = self.sess.get(self.path([ddocid, "_info"])) 81 | r.raise_for_status() 82 | return r.json() 83 | 84 | def create_index(self, fields, idx_type="json", name=None, ddoc=None): 85 | body = { 86 | "index": { 87 | "fields": fields 88 | }, 89 | "type": idx_type, 90 | "w": 3 91 | } 92 | if name is not None: 93 | body["name"] = name 94 | if ddoc is not None: 95 | body["ddoc"] = ddoc 96 | body = json.dumps(body) 97 | r = self.sess.post(self.path("_index"), data=body) 98 | r.raise_for_status() 99 | assert r.json()["id"] is not None 100 | assert r.json()["name"] is not None 101 | return r.json()["result"] == "created" 102 | 103 | def create_text_index(self, analyzer=None, selector=None, idx_type="text", 104 | default_field=None, fields=None, name=None, ddoc=None,index_array_lengths=None): 105 | body = { 106 | "index": { 107 | }, 108 | "type": idx_type, 109 | "w": 3, 110 | } 111 | if name is not None: 112 | body["name"] = name 113 | if analyzer is not None: 114 | body["index"]["default_analyzer"] = analyzer 115 | if default_field is not None: 116 | body["index"]["default_field"] = default_field 117 | if index_array_lengths is not None: 118 | body["index"]["index_array_lengths"] = index_array_lengths 119 | if selector is not None: 120 | body["selector"] = selector 121 | if fields is not None: 122 | body["index"]["fields"] = fields 123 | if ddoc is not None: 124 | body["ddoc"] = ddoc 125 | body = json.dumps(body) 126 | r = self.sess.post(self.path("_index"), data=body) 127 | r.raise_for_status() 128 | return r.json()["result"] == "created" 129 | 130 | def list_indexes(self): 131 | r = self.sess.get(self.path("_index")) 132 | r.raise_for_status() 133 | return r.json()["indexes"] 134 | 135 | def delete_index(self, ddocid, name, idx_type="json"): 136 | path = ["_index", ddocid, idx_type, name] 137 | r = self.sess.delete(self.path(path), params={"w":"3"}) 138 | r.raise_for_status() 139 | 140 | def bulk_delete(self, docs): 141 | body = { 142 | "docids" : docs, 143 | "w": 3 144 | } 145 | body = json.dumps(body) 146 | r = self.sess.post(self.path("_index/_bulk_delete"), data=body) 147 | return r.json() 148 | 149 | def find(self, selector, limit=25, skip=0, sort=None, fields=None, 150 | r=1, conflicts=False, use_index=None, explain=False, 151 | bookmark=None, return_raw=False): 152 | body = { 153 | "selector": selector, 154 | "use_index": use_index, 155 | "limit": limit, 156 | "skip": skip, 157 | "r": r, 158 | "conflicts": conflicts 159 | } 160 | if sort is not None: 161 | body["sort"] = sort 162 | if fields is not None: 163 | body["fields"] = fields 164 | if bookmark is not None: 165 | body["bookmark"] = bookmark 166 | body = json.dumps(body) 167 | if explain: 168 | path = self.path("_explain") 169 | else: 170 | path = self.path("_find") 171 | r = self.sess.post(path, data=body) 172 | r.raise_for_status() 173 | if explain or return_raw: 174 | return r.json() 175 | else: 176 | return r.json()["docs"] 177 | 178 | def find_one(self, *args, **kwargs): 179 | results = self.find(*args, **kwargs) 180 | if len(results) > 1: 181 | raise RuntimeError("Multiple results for Database.find_one") 182 | if len(results): 183 | return results[0] 184 | else: 185 | return None 186 | 187 | 188 | class DbPerClass(unittest.TestCase): 189 | 190 | @classmethod 191 | def setUpClass(klass): 192 | klass.db = Database("127.0.0.1", "5984", random_db_name()) 193 | klass.db.create(q=1, n=3) 194 | 195 | def setUp(self): 196 | self.db = self.__class__.db 197 | 198 | 199 | class UserDocsTests(DbPerClass): 200 | 201 | @classmethod 202 | def setUpClass(klass): 203 | super(UserDocsTests, klass).setUpClass() 204 | user_docs.setup(klass.db) 205 | 206 | 207 | class UserDocsTextTests(DbPerClass): 208 | 209 | DEFAULT_FIELD = None 210 | FIELDS = None 211 | 212 | @classmethod 213 | def setUpClass(klass): 214 | super(UserDocsTextTests, klass).setUpClass() 215 | user_docs.setup( 216 | klass.db, 217 | index_type="text", 218 | default_field=klass.DEFAULT_FIELD, 219 | fields=klass.FIELDS 220 | ) 221 | 222 | 223 | class FriendDocsTextTests(DbPerClass): 224 | 225 | @classmethod 226 | def setUpClass(klass): 227 | super(FriendDocsTextTests, klass).setUpClass() 228 | friend_docs.setup(klass.db, index_type="text") 229 | 230 | class LimitDocsTextTests(DbPerClass): 231 | 232 | @classmethod 233 | def setUpClass(klass): 234 | super(LimitDocsTextTests, klass).setUpClass() 235 | limit_docs.setup(klass.db, index_type="text") 236 | 237 | class NumStringDocsTextTests(DbPerClass): 238 | 239 | @classmethod 240 | def setUpClass(klass): 241 | super(NumStringDocsTextTests, klass).setUpClass() 242 | num_string_docs.setup(klass.db, index_type="text") 243 | -------------------------------------------------------------------------------- /test/user_docs.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 3 | # use this file except in compliance with the License. You may obtain a copy of 4 | # the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 10 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 11 | # License for the specific language governing permissions and limitations under 12 | # the License. 13 | 14 | """ 15 | Generated with http://www.json-generator.com/ 16 | 17 | With this pattern: 18 | 19 | [ 20 | '{{repeat(20)}}', 21 | { 22 | _id: '{{guid()}}', 23 | user_id: "{{index()}}", 24 | name: { 25 | first: "{{firstName()}}", 26 | last: "{{surname()}}" 27 | }, 28 | age: "{{integer(18,90)}}", 29 | location: { 30 | state: "{{state()}}", 31 | city: "{{city()}}", 32 | address: { 33 | street: "{{street()}}", 34 | number: "{{integer(10, 10000)}}" 35 | } 36 | }, 37 | company: "{{company()}}", 38 | email: "{{email()}}", 39 | manager: "{{bool()}}", 40 | twitter: function(tags) { 41 | if(this.manager) 42 | return; 43 | return "@" + this.email.split("@")[0]; 44 | }, 45 | favorites: [ 46 | "{{repeat(2,5)}}", 47 | "{{random('C', 'C++', 'Python', 'Ruby', 'Erlang', 'Lisp')}}" 48 | ] 49 | } 50 | ] 51 | """ 52 | 53 | 54 | import copy 55 | 56 | 57 | def setup(db, index_type="view", **kwargs): 58 | db.recreate() 59 | db.save_docs(copy.deepcopy(DOCS)) 60 | if index_type == "view": 61 | add_view_indexes(db, kwargs) 62 | elif index_type == "text": 63 | add_text_indexes(db, kwargs) 64 | 65 | 66 | def add_view_indexes(db, kwargs): 67 | indexes = [ 68 | ["user_id"], 69 | ["name.last", "name.first"], 70 | ["age"], 71 | [ 72 | "location.state", 73 | "location.city", 74 | "location.address.street", 75 | "location.address.number" 76 | ], 77 | ["company", "manager"], 78 | ["manager"], 79 | ["favorites"], 80 | ["favorites.3"], 81 | ["twitter"] 82 | ] 83 | for idx in indexes: 84 | assert db.create_index(idx) is True 85 | 86 | 87 | def add_text_indexes(db, kwargs): 88 | db.create_text_index(**kwargs) 89 | 90 | 91 | DOCS = [ 92 | { 93 | "_id": "71562648-6acb-42bc-a182-df6b1f005b09", 94 | "user_id": 0, 95 | "name": { 96 | "first": "Stephanie", 97 | "last": "Kirkland" 98 | }, 99 | "age": 48, 100 | "location": { 101 | "state": "Nevada", 102 | "city": "Ronco", 103 | "address": { 104 | "street": "Evergreen Avenue", 105 | "number": 347 106 | } 107 | }, 108 | "company": "Dreamia", 109 | "email": "stephaniekirkland@dreamia.com", 110 | "manager": False, 111 | "twitter": "@stephaniekirkland", 112 | "favorites": [ 113 | "Ruby", 114 | "C", 115 | "Python" 116 | ], 117 | "test" : [{"a":1}, {"b":2}] 118 | }, 119 | { 120 | "_id": "12a2800c-4fe2-45a8-8d78-c084f4e242a9", 121 | "user_id": 1, 122 | "name": { 123 | "first": "Abbott", 124 | "last": "Watson" 125 | }, 126 | "age": 31, 127 | "location": { 128 | "state": "Connecticut", 129 | "city": "Gerber", 130 | "address": { 131 | "street": "Huntington Street", 132 | "number": 8987 133 | } 134 | }, 135 | "company": "Talkola", 136 | "email": "abbottwatson@talkola.com", 137 | "manager": False, 138 | "twitter": "@abbottwatson", 139 | "favorites": [ 140 | "Ruby", 141 | "Python", 142 | "C", 143 | {"Versions": {"Alpha": "Beta"}} 144 | ], 145 | "test" : [{"a":1, "b":2}] 146 | }, 147 | { 148 | "_id": "48ca0455-8bd0-473f-9ae2-459e42e3edd1", 149 | "user_id": 2, 150 | "name": { 151 | "first": "Shelly", 152 | "last": "Ewing" 153 | }, 154 | "age": 42, 155 | "location": { 156 | "state": "New Mexico", 157 | "city": "Thornport", 158 | "address": { 159 | "street": "Miller Avenue", 160 | "number": 7100 161 | } 162 | }, 163 | "company": "Zialactic", 164 | "email": "shellyewing@zialactic.com", 165 | "manager": True, 166 | "favorites": [ 167 | "Lisp", 168 | "Python", 169 | "Erlang" 170 | ], 171 | "test_in": {"val1" : 1, "val2": "val2"} 172 | }, 173 | { 174 | "_id": "0461444c-e60a-457d-a4bb-b8d811853f21", 175 | "user_id": 3, 176 | "name": { 177 | "first": "Madelyn", 178 | "last": "Soto" 179 | }, 180 | "age": 79, 181 | "location": { 182 | "state": "Utah", 183 | "city": "Albany", 184 | "address": { 185 | "street": "Stockholm Street", 186 | "number": 710 187 | } 188 | }, 189 | "company": "Tasmania", 190 | "email": "madelynsoto@tasmania.com", 191 | "manager": True, 192 | "favorites": [[ 193 | "Lisp", 194 | "Erlang", 195 | "Python" 196 | ], 197 | "Erlang", 198 | "C", 199 | "Erlang" 200 | ], 201 | "11111": "number_field", 202 | "22222": {"33333" : "nested_number_field"} 203 | }, 204 | { 205 | "_id": "8e1c90c0-ac18-4832-8081-40d14325bde0", 206 | "user_id": 4, 207 | "name": { 208 | "first": "Nona", 209 | "last": "Horton" 210 | }, 211 | "age": 61, 212 | "location": { 213 | "state": "Georgia", 214 | "city": "Corinne", 215 | "address": { 216 | "street": "Woodhull Street", 217 | "number": 6845 218 | } 219 | }, 220 | "company": "Signidyne", 221 | "email": "nonahorton@signidyne.com", 222 | "manager": False, 223 | "twitter": "@nonahorton", 224 | "favorites": [ 225 | "Lisp", 226 | "C", 227 | "Ruby", 228 | "Ruby" 229 | ], 230 | "name.first" : "name dot first" 231 | }, 232 | { 233 | "_id": "a33d5457-741a-4dce-a217-3eab28b24e3e", 234 | "user_id": 5, 235 | "name": { 236 | "first": "Sheri", 237 | "last": "Perkins" 238 | }, 239 | "age": 73, 240 | "location": { 241 | "state": "Michigan", 242 | "city": "Nutrioso", 243 | "address": { 244 | "street": "Bassett Avenue", 245 | "number": 5648 246 | } 247 | }, 248 | "company": "Myopium", 249 | "email": "sheriperkins@myopium.com", 250 | "manager": True, 251 | "favorites": [ 252 | "Lisp", 253 | "Lisp" 254 | ] 255 | }, 256 | { 257 | "_id": "b31dad3f-ae8b-4f86-8327-dfe8770beb27", 258 | "user_id": 6, 259 | "name": { 260 | "first": "Tate", 261 | "last": "Guy" 262 | }, 263 | "age": 47, 264 | "location": { 265 | "state": "Illinois", 266 | "city": "Helen", 267 | "address": { 268 | "street": "Schenck Court", 269 | "number": 7392 270 | } 271 | }, 272 | "company": "Prosely", 273 | "email": "tateguy@prosely.com", 274 | "manager": True, 275 | "favorites": [ 276 | "C", 277 | "Lisp", 278 | "Ruby", 279 | "C" 280 | ] 281 | }, 282 | { 283 | "_id": "659d0430-b1f4-413a-a6b7-9ea1ef071325", 284 | "user_id": 7, 285 | "name": { 286 | "first": "Jewell", 287 | "last": "Stafford" 288 | }, 289 | "age": 33, 290 | "location": { 291 | "state": "Iowa", 292 | "city": "Longbranch", 293 | "address": { 294 | "street": "Dodworth Street", 295 | "number": 3949 296 | } 297 | }, 298 | "company": "Niquent", 299 | "email": "jewellstafford@niquent.com", 300 | "manager": True, 301 | "favorites": [ 302 | "C", 303 | "C", 304 | "Ruby", 305 | "Ruby", 306 | "Erlang" 307 | ], 308 | "exists_field" : "should_exist1" 309 | 310 | }, 311 | { 312 | "_id": "6c0afcf1-e57e-421d-a03d-0c0717ebf843", 313 | "user_id": 8, 314 | "name": { 315 | "first": "James", 316 | "last": "Mcdaniel" 317 | }, 318 | "age": 68, 319 | "location": { 320 | "state": "Maine", 321 | "city": "Craig", 322 | "address": { 323 | "street": "Greene Avenue", 324 | "number": 8776 325 | } 326 | }, 327 | "company": "Globoil", 328 | "email": "jamesmcdaniel@globoil.com", 329 | "manager": True, 330 | "favorites": None, 331 | "exists_field" : "should_exist2" 332 | }, 333 | { 334 | "_id": "954272af-d5ed-4039-a5eb-8ed57e9def01", 335 | "user_id": 9, 336 | "name": { 337 | "first": "Ramona", 338 | "last": "Floyd" 339 | }, 340 | "age": 22, 341 | "location": { 342 | "state": "Missouri", 343 | "city": "Foxworth", 344 | "address": { 345 | "street": "Lott Place", 346 | "number": 1697 347 | } 348 | }, 349 | "company": "Manglo", 350 | "email": "ramonafloyd@manglo.com", 351 | "manager": True, 352 | "favorites": [ 353 | "Lisp", 354 | "Erlang", 355 | "Python" 356 | ], 357 | "exists_array" : ["should", "exist", "array1"], 358 | "complex_field_value" : "+-(){}[]^~&&*||\"\\/?:!" 359 | }, 360 | { 361 | "_id": "e900001d-bc48-48a6-9b1a-ac9a1f5d1a03", 362 | "user_id": 10, 363 | "name": { 364 | "first": "Charmaine", 365 | "last": "Mills" 366 | }, 367 | "age": 43, 368 | "location": { 369 | "state": "New Hampshire", 370 | "city": "Kiskimere", 371 | "address": { 372 | "street": "Nostrand Avenue", 373 | "number": 4503 374 | } 375 | }, 376 | "company": "Lyria", 377 | "email": "charmainemills@lyria.com", 378 | "manager": True, 379 | "favorites": [ 380 | "Erlang", 381 | "Erlang" 382 | ], 383 | "exists_array" : ["should", "exist", "array2"] 384 | }, 385 | { 386 | "_id": "b06aadcf-cd0f-4ca6-9f7e-2c993e48d4c4", 387 | "user_id": 11, 388 | "name": { 389 | "first": "Mathis", 390 | "last": "Hernandez" 391 | }, 392 | "age": 75, 393 | "location": { 394 | "state": "Hawaii", 395 | "city": "Dupuyer", 396 | "address": { 397 | "street": "Bancroft Place", 398 | "number": 2741 399 | } 400 | }, 401 | "company": "Affluex", 402 | "email": "mathishernandez@affluex.com", 403 | "manager": True, 404 | "favorites": [ 405 | "Ruby", 406 | "Lisp", 407 | "C", 408 | "C++", 409 | "C++" 410 | ], 411 | "exists_object" : {"should": "object"} 412 | }, 413 | { 414 | "_id": "5b61abc1-a3d3-4092-b9d7-ced90e675536", 415 | "user_id": 12, 416 | "name": { 417 | "first": "Patti", 418 | "last": "Rosales" 419 | }, 420 | "age": 71, 421 | "location": { 422 | "state": "Pennsylvania", 423 | "city": "Juntura", 424 | "address": { 425 | "street": "Hunterfly Place", 426 | "number": 7683 427 | } 428 | }, 429 | "company": "Oulu", 430 | "email": "pattirosales@oulu.com", 431 | "manager": True, 432 | "favorites": [ 433 | "C", 434 | "Python", 435 | "Lisp" 436 | ], 437 | "exists_object" : {"another": "object"} 438 | }, 439 | { 440 | "_id": "b1e70402-8add-4068-af8f-b4f3d0feb049", 441 | "user_id": 13, 442 | "name": { 443 | "first": "Whitley", 444 | "last": "Harvey" 445 | }, 446 | "age": 78, 447 | "location": { 448 | "state": "Minnesota", 449 | "city": "Trail", 450 | "address": { 451 | "street": "Pleasant Place", 452 | "number": 8766 453 | } 454 | }, 455 | "company": None, 456 | "email": "whitleyharvey@fangold.com", 457 | "manager": False, 458 | "twitter": "@whitleyharvey", 459 | "favorites": [ 460 | "C", 461 | "Ruby", 462 | "Ruby" 463 | ] 464 | }, 465 | { 466 | "_id": "c78c529f-0b07-4947-90a6-d6b7ca81da62", 467 | "user_id": 14, 468 | "name": { 469 | "first": "Faith", 470 | "last": "Hess" 471 | }, 472 | "age": 51, 473 | "location": { 474 | "state": "North Dakota", 475 | "city": "Axis", 476 | "address": { 477 | "street": "Brightwater Avenue", 478 | "number": 1106 479 | } 480 | }, 481 | "company": "Pharmex", 482 | "email": "faithhess@pharmex.com", 483 | "manager": True, 484 | "favorites": [ 485 | "Erlang", 486 | "Python", 487 | "Lisp" 488 | ] 489 | } 490 | ] 491 | --------------------------------------------------------------------------------