├── CONTRIB.md ├── LICENSE ├── README.md ├── java ├── BUILD ├── COPYING ├── README.md ├── build.xml ├── src │ └── com │ │ └── google │ │ └── appengine │ │ └── demos │ │ └── search │ │ └── TextSearchServlet.java └── war │ ├── WEB-INF │ ├── appengine-web.xml │ ├── lib │ │ └── README │ ├── logging.properties │ └── web.xml │ ├── favicon.ico │ ├── index.jsp │ └── redirect.jsp ├── product_search_python ├── README.md ├── admin.py ├── admin_handlers.py ├── app.yaml ├── base_handler.py ├── categories.py ├── config.py ├── cron.yaml ├── data │ ├── sample_data_books.csv │ ├── sample_data_books_update.csv │ └── sample_data_tvs.csv ├── docs.py ├── errors.py ├── handlers.py ├── main.py ├── models.py ├── queue.yaml ├── sortoptions.py ├── static │ ├── css │ │ ├── bootstrap.css │ │ ├── byword.css │ │ └── main.css │ ├── instrs.html │ └── js │ │ └── StyledMarker.js ├── stores.py ├── templates │ ├── admin.html │ ├── base.html │ ├── create_product.html │ ├── index.html │ ├── notification.html │ ├── product.html │ ├── review.html │ └── reviews.html ├── tests │ ├── __init__.py │ ├── run_unittests.py │ ├── test_data │ │ └── testdata.csv │ ├── test_errors.py │ └── test_search.py └── utils.py └── python ├── app.yaml ├── search_demo.py ├── static └── css │ └── main.css └── templates └── index.html /CONTRIB.md: -------------------------------------------------------------------------------- 1 | # How to become a contributor and submit your own code 2 | 3 | ## Contributor License Agreements 4 | 5 | We'd love to accept your sample apps and patches! Before we can take them, we have to jump a couple of legal hurdles. 6 | 7 | Please fill out either the individual or corporate Contributor License Agreement (CLA). 8 | 9 | * If you are an individual writing original source code and you're sure you own the intellectual property, then you'll need to sign an [individual CLA](http://code.google.com/legal/individual-cla-v1.0.html). 10 | * If you work for a company that wants to allow you to contribute your work, then you'll need to sign a [corporate CLA](http://code.google.com/legal/corporate-cla-v1.0.html). 11 | 12 | Follow either of the two links above to access the appropriate CLA and instructions for how to sign and return it. Once we receive it, we'll be able to accept your pull requests. 13 | 14 | ## Contributing A Patch 15 | 16 | 1. Submit an issue describing your proposed change to the repo in question. 17 | 1. The repo owner will respond to your issue promptly. 18 | 1. If your proposed change is accepted, and you haven't already done so, sign a Contributor License Agreement (see details above). 19 | 1. Fork the desired repo, develop and test your code changes. 20 | 1. Ensure that your code adheres to the existing style in the sample to which you are contributing. Refer to the [Google Cloud Platform Samples Style Guide](https://github.com/GoogleCloudPlatform/Template/wiki/style.html) for the recommended coding standards for this organization. 21 | 1. Ensure that your code has an appropriate set of unit tests which all pass. 22 | 1. Submit a pull request. 23 | 24 | ## Contributing A New Sample App 25 | 26 | 1. Submit an issue to the GoogleCloudPlatform/Template repo describing your proposed sample app. 27 | 1. The Template repo owner will respond to your enhancement issue promptly. Instructional value is the top priority when evaluating new app proposals for this collection of repos. 28 | 1. If your proposal is accepted, and you haven't already done so, sign a Contributor License Agreement (see details above). 29 | 1. Create your own repo for your app following this naming convention: 30 | * {product}-{app-name}-{language} 31 | * products: appengine, compute, storage, bigquery, prediction, cloudsql 32 | * example: appengine-guestbook-python 33 | * For multi-product apps, concatenate the primary products, like this: compute-appengine-demo-suite-python. 34 | * For multi-language apps, concatenate the primary languages like this: appengine-sockets-python-java-go. 35 | 1. Clone the README.md, CONTRIB.md and LICENSE files from the GoogleCloudPlatform/Template repo. 36 | 1. Ensure that your code adheres to the existing style in the sample to which you are contributing. Refer to the [Google Cloud Platform Samples Style Guide](https://github.com/GoogleCloudPlatform/Template/wiki/style.html) for the recommended coding standards for this organization. 37 | 1. Ensure that your code has an appropriate set of unit tests which all pass. 38 | 1. Submit a request to fork your repo in GoogleCloudPlatform organizationt via your proposal issue. 39 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | App Engine Search API examples 2 | ============================ 3 | 4 | A set of examples showing how to use the App Engine Search API. 5 | 6 | There are two simple examples, one written in Java (the `java` directory) and one written in Python (the `python`) directory. 7 | The `product_search_python` directory holds a more complex example— a product search app— written in Python. 8 | 9 | -------------------------------------------------------------------------------- /java/BUILD: -------------------------------------------------------------------------------- 1 | # -*- mode: python; -*- 2 | # 3 | # Copyright 2011 Google Inc. All Rights Reserved. 4 | # 5 | 6 | package(default_visibility = ['//visibility:private']) 7 | 8 | Fileset( 9 | name = 'src', 10 | out = 'search', 11 | entries = [ 12 | FilesetEntry(excludes=[ 13 | 'BUILD', 14 | '.*~', 15 | ]), 16 | ]) 17 | -------------------------------------------------------------------------------- /java/COPYING: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | 1. Definitions. 6 | 7 | "License" shall mean the terms and conditions for use, reproduction, and 8 | distribution as defined by Sections 1 through 9 of this document. 9 | 10 | "Licensor" shall mean the copyright owner or entity authorized by the 11 | copyright owner that is granting the License. 12 | 13 | "Legal Entity" shall mean the union of the acting entity and all other 14 | entities that control, are controlled by, or are under common control with 15 | that entity. For the purposes of this definition, "control" means (i) the 16 | power, direct or indirect, to cause the direction or management of such 17 | entity, whether by contract or otherwise, or (ii) ownership of fifty percent 18 | (50%) or more of the outstanding shares, or (iii) beneficial ownership of 19 | such entity. 20 | 21 | "You" (or "Your") shall mean an individual or Legal Entity exercising 22 | permissions granted by this License. 23 | 24 | "Source" form shall mean the preferred form for making modifications, 25 | including but not limited to software source code, documentation source, and 26 | configuration files. 27 | 28 | "Object" form shall mean any form resulting from mechanical transformation 29 | or translation of a Source form, including but not limited to compiled 30 | object code, generated documentation, and conversions to other media types. 31 | 32 | "Work" shall mean the work of authorship, whether in Source or Object form, 33 | made available under the License, as indicated by a copyright notice that is 34 | included in or attached to the work (an example is provided in the Appendix 35 | below). 36 | 37 | "Derivative Works" shall mean any work, whether in Source or Object form, 38 | that is based on (or derived from) the Work and for which the editorial 39 | revisions, annotations, elaborations, or other modifications represent, as a 40 | whole, an original work of authorship. For the purposes of this License, 41 | Derivative Works shall not include works that remain separable from, or 42 | merely link (or bind by name) to the interfaces of, the Work and Derivative 43 | Works thereof. 44 | 45 | "Contribution" shall mean any work of authorship, including the original 46 | version of the Work and any modifications or additions to that Work or 47 | Derivative Works thereof, that is intentionally submitted to Licensor for 48 | inclusion in the Work by the copyright owner or by an individual or Legal 49 | Entity authorized to submit on behalf of the copyright owner. For the 50 | purposes of this definition, "submitted" means any form of electronic, 51 | verbal, or written communication sent to the Licensor or its 52 | representatives, including but not limited to communication on electronic 53 | mailing lists, source code control systems, and issue tracking systems that 54 | are managed by, or on behalf of, the Licensor for the purpose of discussing 55 | and improving the Work, but excluding communication that is conspicuously 56 | marked or otherwise designated in writing by the copyright owner as "Not a 57 | Contribution." 58 | 59 | "Contributor" shall mean Licensor and any individual or Legal Entity on 60 | behalf of whom a Contribution has been received by Licensor and subsequently 61 | incorporated within the Work. 62 | 63 | 2. Grant of Copyright License. Subject to the terms and conditions of this 64 | License, each Contributor hereby grants to You a perpetual, worldwide, 65 | non-exclusive, no-charge, royalty-free, irrevocable copyright license to 66 | reproduce, prepare Derivative Works of, publicly display, publicly perform, 67 | sublicense, and distribute the Work and such Derivative Works in Source or 68 | Object form. 69 | 70 | 3. Grant of Patent License. Subject to the terms and conditions of this 71 | License, each Contributor hereby grants to You a perpetual, worldwide, 72 | non-exclusive, no-charge, royalty-free, irrevocable (except as stated in 73 | this section) patent license to make, have made, use, offer to sell, sell, 74 | import, and otherwise transfer the Work, where such license applies only to 75 | those patent claims licensable by such Contributor that are necessarily 76 | infringed by their Contribution(s) alone or by combination of their 77 | Contribution(s) with the Work to which such Contribution(s) was submitted. 78 | If You institute patent litigation against any entity (including a 79 | cross-claim or counterclaim in a lawsuit) alleging that the Work or a 80 | Contribution incorporated within the Work constitutes direct or contributory 81 | patent infringement, then any patent licenses granted to You under this 82 | License for that Work shall terminate as of the date such litigation is 83 | filed. 84 | 85 | 4. Redistribution. You may reproduce and distribute copies of the Work or 86 | Derivative Works thereof in any medium, with or without modifications, and 87 | in Source or Object form, provided that You meet the following conditions: 88 | 89 | a. You must give any other recipients of the Work or Derivative Works a copy 90 | of this License; and 91 | 92 | b. You must cause any modified files to carry prominent notices stating that 93 | You changed the files; and 94 | 95 | c. You must retain, in the Source form of any Derivative Works that You 96 | distribute, all copyright, patent, trademark, and attribution notices from 97 | the Source form of the Work, excluding those notices that do not pertain to 98 | any part of the Derivative Works; and 99 | 100 | d. If the Work includes a "NOTICE" text file as part of its distribution, 101 | then any Derivative Works that You distribute must include a readable copy 102 | of the attribution notices contained within such NOTICE file, excluding 103 | those notices that do not pertain to any part of the Derivative Works, in at 104 | least one of the following places: within a NOTICE text file distributed as 105 | part of the Derivative Works; within the Source form or documentation, if 106 | provided along with the Derivative Works; or, within a display generated by 107 | the Derivative Works, if and wherever such third-party notices normally 108 | appear. The contents of the NOTICE file are for informational purposes only 109 | and do not modify the License. You may add Your own attribution notices 110 | within Derivative Works that You distribute, alongside or as an addendum to 111 | the NOTICE text from the Work, provided that such additional attribution 112 | notices cannot be construed as modifying the License. 113 | 114 | You may add Your own copyright statement to Your modifications and may 115 | provide additional or different license terms and conditions for use, 116 | reproduction, or distribution of Your modifications, or for any such 117 | Derivative Works as a whole, provided Your use, reproduction, and 118 | distribution of the Work otherwise complies with the conditions stated in 119 | this License. 120 | 121 | 5. Submission of Contributions. Unless You explicitly state otherwise, any 122 | Contribution intentionally submitted for inclusion in the Work by You to the 123 | Licensor shall be under the terms and conditions of this License, without 124 | any additional terms or conditions. Notwithstanding the above, nothing 125 | herein shall supersede or modify the terms of any separate license agreement 126 | you may have executed with Licensor regarding such Contributions. 127 | 128 | 6. Trademarks. This License does not grant permission to use the trade 129 | names, trademarks, service marks, or product names of the Licensor, except 130 | as required for reasonable and customary use in describing the origin of the 131 | Work and reproducing the content of the NOTICE file. 132 | 133 | 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in 134 | writing, Licensor provides the Work (and each Contributor provides its 135 | Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 136 | KIND, either express or implied, including, without limitation, any 137 | warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or 138 | FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining 139 | the appropriateness of using or redistributing the Work and assume any risks 140 | associated with Your exercise of permissions under this License. 141 | 142 | 8. Limitation of Liability. In no event and under no legal theory, whether 143 | in tort (including negligence), contract, or otherwise, unless required by 144 | applicable law (such as deliberate and grossly negligent acts) or agreed to 145 | in writing, shall any Contributor be liable to You for damages, including 146 | any direct, indirect, special, incidental, or consequential damages of any 147 | character arising as a result of this License or out of the use or inability 148 | to use the Work (including but not limited to damages for loss of goodwill, 149 | work stoppage, computer failure or malfunction, or any and all other 150 | commercial damages or losses), even if such Contributor has been advised of 151 | the possibility of such damages. 152 | 153 | 9. Accepting Warranty or Additional Liability. While redistributing the Work 154 | or Derivative Works thereof, You may choose to offer, and charge a fee for, 155 | acceptance of support, warranty, indemnity, or other liability obligations 156 | and/or rights consistent with this License. However, in accepting such 157 | obligations, You may act only on Your own behalf and on Your sole 158 | responsibility, not on behalf of any other Contributor, and only if You 159 | agree to indemnify, defend, and hold each Contributor harmless for any 160 | liability incurred by, or claims asserted against, such Contributor by 161 | reason of your accepting any such warranty or additional liability. 162 | -------------------------------------------------------------------------------- /java/README.md: -------------------------------------------------------------------------------- 1 | 2 | # Google App Engine Java Runtime SDK - Text Search Demo 3 | 4 | Illustrates simple examples that use the Search API. 5 | 6 | To compile and run this sample on the local dev server, edit `build.xml` to point to your copy of the App Engine Java SDK. 7 | Then, run `ant` in this directory as follows: 8 | 9 | ant runserver 10 | 11 | Once the server is running, point your web browser at: 12 | 13 | http://localhost:8080/ 14 | -------------------------------------------------------------------------------- /java/build.xml: -------------------------------------------------------------------------------- 1 | 2 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 26 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 61 | 62 | 63 | 64 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 74 | 75 | 76 | 77 | 79 | 80 | 81 | 82 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | -------------------------------------------------------------------------------- /java/src/com/google/appengine/demos/search/TextSearchServlet.java: -------------------------------------------------------------------------------- 1 | // Copyright 2011 Google Inc. All Rights Reserved. 2 | 3 | package com.google.appengine.demos.search; 4 | 5 | import com.google.appengine.api.search.Document; 6 | import com.google.appengine.api.search.Field; 7 | import com.google.appengine.api.search.Index; 8 | import com.google.appengine.api.search.IndexSpec; 9 | import com.google.appengine.api.search.OperationResult; 10 | import com.google.appengine.api.search.Query; 11 | import com.google.appengine.api.search.QueryOptions; 12 | import com.google.appengine.api.search.Results; 13 | import com.google.appengine.api.search.DeleteException; 14 | import com.google.appengine.api.search.ScoredDocument; 15 | import com.google.appengine.api.search.SearchServiceFactory; 16 | import com.google.appengine.api.search.StatusCode; 17 | import com.google.appengine.api.users.User; 18 | import com.google.appengine.api.users.UserService; 19 | import com.google.appengine.api.users.UserServiceFactory; 20 | 21 | import java.io.IOException; 22 | import java.util.ArrayList; 23 | import java.util.Arrays; 24 | import java.util.Date; 25 | import java.util.Iterator; 26 | import java.util.List; 27 | import java.util.StringTokenizer; 28 | import java.util.logging.Level; 29 | import java.util.logging.Logger; 30 | 31 | import javax.servlet.ServletException; 32 | import javax.servlet.http.HttpServlet; 33 | import javax.servlet.http.HttpServletRequest; 34 | import javax.servlet.http.HttpServletResponse; 35 | 36 | /** 37 | * A demo servlet showing basic text search capabilities. This servlet 38 | * has a single index shared between all users. It illustrates how to 39 | * add, search for and remove documents from the shared index. 40 | * 41 | */ 42 | public class TextSearchServlet extends HttpServlet { 43 | 44 | private static final long serialVersionUID = 1L; 45 | 46 | private static final String VOID_REMOVE = 47 | "Remove failed due to a null doc ID"; 48 | 49 | private static final String VOID_ADD = 50 | "Document not added due to empty content"; 51 | 52 | /** 53 | * The index used by this application. Since we only have one index 54 | * we create one instance only. We build an index with the default 55 | * consistency, which is Consistency.PER_DOCUMENT. These types of 56 | * indexes are most suitable for streams and feeds, and can cope with 57 | * a high rate of updates. 58 | */ 59 | private static final Index INDEX = SearchServiceFactory.getSearchService() 60 | .getIndex(IndexSpec.newBuilder().setName("shared_index")); 61 | 62 | enum Action { 63 | ADD, REMOVE, DEFAULT; 64 | } 65 | 66 | private static final Logger LOG = Logger.getLogger( 67 | TextSearchServlet.class.getName()); 68 | 69 | @Override 70 | public void doGet(HttpServletRequest req, HttpServletResponse resp) 71 | throws IOException, ServletException { 72 | User currentUser = setupUser(req); 73 | String outcome = null; 74 | switch (getAction(req)) { 75 | case ADD: 76 | outcome = add(req, currentUser); 77 | break; 78 | case REMOVE: 79 | outcome = remove(req); 80 | break; 81 | // On DEFAULT we fall through and just execute search below. 82 | } 83 | String searchOutcome = search(req); 84 | if (outcome == null) { 85 | outcome = searchOutcome; 86 | } 87 | req.setAttribute("outcome", outcome); 88 | req.getRequestDispatcher("index.jsp").forward(req, resp); 89 | } 90 | 91 | private User setupUser(HttpServletRequest req) { 92 | UserService userService = UserServiceFactory.getUserService(); 93 | User currentUser = userService.getCurrentUser(); 94 | if (currentUser != null) { 95 | req.setAttribute("authAction", "sign out"); 96 | req.setAttribute("authUrl", 97 | userService.createLogoutURL(req.getRequestURI())); 98 | } else { 99 | currentUser = new User("nobody@example.com", "example.com"); 100 | req.setAttribute("authAction", "sign in"); 101 | req.setAttribute("authUrl", 102 | userService.createLoginURL(req.getRequestURI())); 103 | } 104 | req.setAttribute("nickname", currentUser.getNickname()); 105 | return currentUser; 106 | } 107 | 108 | /** 109 | * Indexes a document built from the current request on behalf of the 110 | * specified user. Each document has three fields in it. The content 111 | * field stores used entered text. The email, and domain are extracted 112 | * from the current user. 113 | */ 114 | private String add(HttpServletRequest req, User currentUser) { 115 | String content = req.getParameter("doc"); 116 | if (content == null || content.isEmpty()) { 117 | LOG.warning(VOID_ADD); 118 | return VOID_ADD; 119 | } 120 | String ratingStr = req.getParameter("rating"); 121 | int rating = 1; 122 | if (ratingStr != null) { 123 | rating = Integer.parseInt(ratingStr); 124 | } 125 | Document.Builder docBuilder = Document.newBuilder() 126 | .addField(Field.newBuilder().setName("content").setText(content)) 127 | .addField(Field.newBuilder().setName("email") 128 | .setText(currentUser.getEmail())) 129 | .addField(Field.newBuilder().setName("domain") 130 | .setText(currentUser.getAuthDomain())) 131 | .addField(Field.newBuilder().setName("published").setDate( 132 | Field.date(new Date()))) 133 | .addField(Field.newBuilder().setName("rating") 134 | .setNumber(rating)); 135 | String tagStr = req.getParameter("tags"); 136 | if (tagStr != null) { 137 | StringTokenizer tokenizer = new StringTokenizer(tagStr, ","); 138 | while (tokenizer.hasMoreTokens()) { 139 | docBuilder.addField(Field.newBuilder().setName("tag") 140 | .setAtom(tokenizer.nextToken())); 141 | } 142 | } 143 | Document doc = docBuilder.build(); 144 | LOG.info("Adding document:\n" + doc.toString()); 145 | try { 146 | INDEX.put(doc); 147 | return "Document added"; 148 | } catch (RuntimeException e) { 149 | LOG.log(Level.SEVERE, "Failed to add " + doc, e); 150 | return "Document not added due to an error " + e.getMessage(); 151 | } 152 | } 153 | 154 | private String getOnlyField(Document doc, String fieldName, String defaultValue) { 155 | if (doc.getFieldCount(fieldName) == 1) { 156 | return doc.getOnlyField(fieldName).getText(); 157 | } 158 | LOG.severe("Field " + fieldName + " present " + doc.getFieldCount(fieldName)); 159 | return defaultValue; 160 | } 161 | 162 | /** 163 | * Searches the index for matching documents. If the query is not specified 164 | * in the request, we search for any documents. 165 | */ 166 | private String search(HttpServletRequest req) { 167 | String queryStr = req.getParameter("query"); 168 | if (queryStr == null) { 169 | queryStr = ""; 170 | } 171 | String limitStr = req.getParameter("limit"); 172 | int limit = 10; 173 | if (limitStr != null) { 174 | try { 175 | limit = Integer.parseInt(limitStr); 176 | } catch (NumberFormatException e) { 177 | LOG.severe("Failed to parse " + limitStr); 178 | } 179 | } 180 | List found = new ArrayList(); 181 | String outcome = null; 182 | try { 183 | // Rather than just using a query we build a search request. 184 | // This allows us to specify other attributes, such as the 185 | // number of documents to be returned by search. 186 | Query query = Query.newBuilder() 187 | .setOptions(QueryOptions.newBuilder() 188 | .setLimit(limit). 189 | // for deployed apps, uncomment the line below to demo snippeting. 190 | // This will not work on the dev_appserver. 191 | // setFieldsToSnippet("content"). 192 | build()) 193 | .build(queryStr); 194 | LOG.info("Sending query " + query); 195 | Results results = INDEX.search(query); 196 | for (ScoredDocument scoredDoc : results) { 197 | User author = new User( 198 | getOnlyField(scoredDoc, "email", "user"), 199 | getOnlyField(scoredDoc, "domain", "example.com")); 200 | // Rather than presenting the original document to the 201 | // user, we build a derived one that holds author's nickname. 202 | List expressions = scoredDoc.getExpressions(); 203 | String content = null; 204 | if (expressions != null) { 205 | for (Field field : expressions) { 206 | if ("content".equals(field.getName())) { 207 | content = field.getHTML(); 208 | break; 209 | } 210 | } 211 | } 212 | if (content == null) { 213 | content = getOnlyField(scoredDoc, "content", ""); 214 | } 215 | Document derived = Document.newBuilder() 216 | .setId(scoredDoc.getId()) 217 | .addField(Field.newBuilder().setName("content").setText(content)) 218 | .addField(Field.newBuilder().setName("nickname").setText( 219 | author.getNickname())) 220 | .addField(Field.newBuilder().setName("published").setDate( 221 | scoredDoc.getOnlyField("published").getDate())) 222 | .build(); 223 | found.add(derived); 224 | } 225 | } catch (RuntimeException e) { 226 | LOG.log(Level.SEVERE, "Search with query '" + queryStr + "' failed", e); 227 | outcome = "Search failed due to an error: " + e.getMessage(); 228 | } 229 | req.setAttribute("found", found); 230 | return outcome; 231 | } 232 | 233 | /** 234 | * Removes documents with IDs specified in the given request. In the demo 235 | * application we do not perform any authorization checks, thus no user 236 | * information is necessary. 237 | */ 238 | private String remove(HttpServletRequest req) { 239 | String[] docIds = req.getParameterValues("docid"); 240 | if (docIds == null) { 241 | LOG.warning(VOID_REMOVE); 242 | return VOID_REMOVE; 243 | } 244 | List docIdList = Arrays.asList(docIds); 245 | try { 246 | INDEX.delete(docIdList); 247 | return "Documents " + docIdList + " removed"; 248 | } catch (DeleteException e) { 249 | List failedIds = findFailedIds(docIdList, e.getResults()); 250 | LOG.log(Level.SEVERE, "Failed to remove documents " + failedIds, e); 251 | return "Remove failed for " + failedIds; 252 | } 253 | } 254 | 255 | /** 256 | * A convenience method that correlates document status to the document ID. 257 | */ 258 | private List findFailedIds(List docIdList, 259 | List results) { 260 | List failedIds = new ArrayList(); 261 | Iterator opIter = results.iterator(); 262 | Iterator idIter = docIdList.iterator(); 263 | while (opIter.hasNext() && idIter.hasNext()) { 264 | OperationResult result = opIter.next(); 265 | String docId = idIter.next(); 266 | if (!StatusCode.OK.equals(result.getCode())) { 267 | failedIds.add(docId); 268 | } 269 | } 270 | return failedIds; 271 | } 272 | 273 | /** 274 | * Extracts the type of action stored in the request. We have only three 275 | * types of actions: ADD, REMOVE and DEFAULT. The DEFAULT is included 276 | * to indicate action other than ADD or REMOVE. We do not have a special 277 | * acton for search, as we always execute search. This way we show documents 278 | * that match terms entered in the search box, regardless of the operation. 279 | * 280 | * @param HTTP request received by the servlet 281 | * @return the requested user action, as inferred from the request 282 | */ 283 | private Action getAction(HttpServletRequest req) { 284 | if (req.getParameter("index") != null) { 285 | return Action.ADD; 286 | } 287 | if (req.getParameter("delete") != null) { 288 | return Action.REMOVE; 289 | } 290 | return Action.DEFAULT; 291 | } 292 | } 293 | -------------------------------------------------------------------------------- /java/war/WEB-INF/appengine-web.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | search-java-demo 4 | 1 5 | true 6 | 7 | 8 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /java/war/WEB-INF/lib/README: -------------------------------------------------------------------------------- 1 | Jar files which are uploaded to the server are copied into this directory. 2 | -------------------------------------------------------------------------------- /java/war/WEB-INF/logging.properties: -------------------------------------------------------------------------------- 1 | .level = INFO 2 | -------------------------------------------------------------------------------- /java/war/WEB-INF/web.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 7 | 8 | textsearch 9 | com.google.appengine.demos.search.TextSearchServlet 10 | 11 | 12 | textsearch 13 | /search 14 | 15 | 16 | redirect.jsp 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /java/war/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/appengine-search-python-java/fc8c05e05bc57219e423316dd30d55c3eb4df979/java/war/favicon.ico -------------------------------------------------------------------------------- /java/war/index.jsp: -------------------------------------------------------------------------------- 1 | 2 | <%@ page import="com.google.appengine.api.search.Document" %> 3 | <%@ page import="com.google.appengine.api.search.Field" %> 4 | <%@ page import="java.util.List" %> 5 | <%@ page import="java.text.DateFormat" %> 6 | <% 7 | String outcome = (String) request.getAttribute("outcome"); 8 | if (outcome == null || outcome.isEmpty()) { 9 | outcome = " "; 10 | } 11 | String query = (String) request.getParameter("query"); 12 | if (query == null) { 13 | query = ""; 14 | } 15 | String limit = (String) request.getParameter("limit"); 16 | if (limit == null || limit.isEmpty()) { 17 | limit = "10"; 18 | } 19 | %> 20 | 21 | 22 | Text Search Demo 23 | 50 | 65 | 66 | 67 |
68 |
69 | Text Search Demo 70 |
71 |
72 | <%=request.getAttribute("nickname")%> • 73 | <%=request.getAttribute("authAction")%> 74 |
75 |
76 |
<%=outcome%>
77 |
78 | 80 | 87 |
88 |
89 |
90 | Document: 91 |
92 | 93 |
94 | Rating (3): 95 | 97 |
98 | 100 |
101 | 102 | 103 | 104 |
105 |
106 | <% 107 | List found = (List) request.getAttribute("found"); 108 | if (found != null) { 109 | %> 110 |
111 | 112 | 113 | 114 | 115 | 118 | 119 | 120 | 121 | 122 | <% 123 | if (found.isEmpty()) { 124 | %> 125 | 126 | 127 | 128 | <% 129 | } else { 130 | for (Document doc : found) { 131 | %> 132 | 133 | 136 | 139 | 142 | 145 | 146 | <% 147 | } 148 | } 149 | %> 150 |
116 | 117 | AuthorContentPublished
No matching documents found
134 | 135 | 137 | <%=doc.getOnlyField("nickname").getText()%> 138 | 140 | <%=doc.getOnlyField("content").getText()%> 141 | 143 | <%=DateFormat.getDateInstance().format(doc.getOnlyField("published").getDate())%> 144 |
151 | 152 |
153 | <% 154 | } 155 | %> 156 | 157 | 158 | -------------------------------------------------------------------------------- /java/war/redirect.jsp: -------------------------------------------------------------------------------- 1 | <% 2 | request.getRequestDispatcher("/search").forward(request, response); 3 | %> 4 | -------------------------------------------------------------------------------- /product_search_python/README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Full-Text Search Demo App: Product Search 4 | 5 | ## Introduction 6 | 7 | This Python App Engine application illustrates the use of the [Full-Text Search 8 | API](https://developers.google.com/appengine/docs/python/search) in a "Product 9 | Search" domain with two categories of sample products: *books* and 10 | *hd televisions*. This README assumes that you are already familiar with how to 11 | configure and deploy an App Engine app. If not, first see the App Engine 12 | [documentation](https://developers.google.com/appengine/docs/python/overview) 13 | and [Getting Started guide](https://developers.google.com/appengine/docs/python/gettingstarted). 14 | 15 | This demo app allows users to search product information using full-text search, 16 | and to add reviews with ratings to the products. Search results can be sorted 17 | according to several criteria. In conjunction with listed search results, a 18 | sidebar allows the user to further filter the search results on product rating. 19 | (A product's rating is the average of its reviews, so if a product has no 20 | reviews yet, its rating will be 0). 21 | 22 | A user does not need to be logged in to search the products, or to add reviews. 23 | A user must be logged in **as an admin of the app to add or modify product 24 | data**. The sidebar admin links are not displayed for non-admin users. 25 | 26 | ## Configuration 27 | 28 | Before you deploy the application, edit `app.yaml` to specify your own app id and version. 29 | 30 | In `templates/product.html`, the Google Maps API is accessed. It does not require an API key, but you are encouraged to use one to monitor your maps usage. In the element, look for: 31 | 32 | src="https://maps.googleapis.com/maps/api/js?sensor=false" 33 | 34 | and replace it with something like the following, where `replaceWithYourAPIKey` is your own API key: 35 | 36 | src="https://maps.googleapis.com/maps/api/js?sensor=false&key=replaceWithYourAPIKey" 37 | 38 | as described [here](https://developers.google.com/maps/documentation/javascript/tutorial#api_key). 39 | 40 | ## Information About Running the App Locally 41 | 42 | Log in as an app admin to add and modify the app's product data. 43 | 44 | The app uses XG (cross-group) transactions, which requires the dev_appserver to 45 | be run with the `--high_replication` flag. E.g., to start up the dev_appserver 46 | from the command line in the project directory (this directory), assuming the 47 | GAE SDK is in your path, do: 48 | 49 | dev_appserver.py --high_replication . 50 | 51 | The app is configured to use Python 2.7. On some platforms, it may also be 52 | necessary to have Python 2.7 installed locally when running the dev_appserver. 53 | The app's unit tests also require Python 2.7. 54 | 55 | When running the app locally, not all features of the search API are supported. 56 | So, not all search queries may give the same results during local testing as 57 | when run with the deployed app. 58 | Be sure to test on a deployed version of your app as well as locally. 59 | 60 | ## Administering the deployed app 61 | 62 | You will need to be logged in as an administrator of the app to add and modify 63 | product data, though not to search products or add reviews. If you want to 64 | remove this restriction, you can edit the `login: admin` specification in 65 | `app.yaml`, and remove the `@BaseHandler.admin` decorators in 66 | `admin_handlers.py`. 67 | 68 | ## Loading Sample Data 69 | 70 | When you first start up your app, you will want to add sample data to it. 71 | 72 | Sample product data can be added in two ways. First, sample product data in CSV 73 | format can be added in batch via a link on the app's admin page. Batch indexing 74 | of documents is more efficient than adding the documents one at a time. For consistency, 75 | **the batch addition of sample data first removes all 76 | existing index and datastore product data**. 77 | 78 | The second way to add sample data is via the admin's "Create new product" link 79 | in the sidebar, which lets an admin add sample products (either "books" or 80 | "hd televisions") one at a time. 81 | 82 | 83 | ## Updating product documents with a new average rating 84 | 85 | When a user creates a new review, the average rating for that product is 86 | updated in the datastore. The app may be configured to update the associated 87 | product `search.Document` at the same time (the default), or do this at a 88 | later time in batch (which is more efficient). See `cron.yaml` for an example 89 | of how to do this update periodically in batch. 90 | 91 | ## Searches 92 | 93 | Any valid queries can be typed into the search box. This includes simple word 94 | and phrase queries, but you may also submit queries that include references to 95 | specific document fields and use numeric comparators on numeric fields. See the 96 | Search API's 97 | [documentation](https://developers.google.com/appengine/docs/python/search) for 98 | a description of the query syntax. 99 | 100 | Thus, for explanatory purposes, the "product details" show all actual 101 | field names of the given product document; you can use this information to 102 | construct queries against those fields. In the same spirit, the raw 103 | query string used for the query is displayed with the search results. 104 | 105 | Only product information is searched; product review text is not included in the 106 | search. 107 | 108 | ### Some example searches 109 | 110 | Below are some example product queries, which assume the sample data has been loaded. 111 | As discussed above, not all of these queries are supported by the dev_appserver. 112 | 113 | `stories price < 10` 114 | `price > 10 price < 15` 115 | `publisher:Vintage` 116 | `Mega TVs` 117 | `name:tv1` 118 | `size > 30` 119 | 120 | ## Geosearch 121 | 122 | This application includes an example of using the Search API to perform 123 | location-based queries. Sample store location data is defined in `stores.py`, 124 | and is loaded along with the product data. The product details page for a 125 | product allows a search for stores within a given radius of the user's current 126 | location. The user's location is obtained from the browser. 127 | -------------------------------------------------------------------------------- /product_search_python/admin.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright 2012 Google Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | """Defines the routing for the app's admin request handlers 18 | (those that require administrative access).""" 19 | 20 | from admin_handlers import * 21 | 22 | import webapp2 23 | 24 | application = webapp2.WSGIApplication( 25 | [ 26 | ('/admin/manage', AdminHandler), 27 | ('/admin/create_product', CreateProductHandler), 28 | ('/admin/delete_product', DeleteProductHandler) 29 | ], 30 | debug=True) 31 | 32 | -------------------------------------------------------------------------------- /product_search_python/admin_handlers.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright 2012 Google Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | """ Contains the admin request handlers for the app (those that require 18 | administrative access). 19 | """ 20 | 21 | import csv 22 | import logging 23 | import os 24 | import urllib 25 | import uuid 26 | 27 | from base_handler import BaseHandler 28 | import categories 29 | import config 30 | import docs 31 | import errors 32 | import models 33 | import stores 34 | import utils 35 | 36 | from google.appengine.api import users 37 | from google.appengine.ext.deferred import defer 38 | from google.appengine.ext import ndb 39 | from google.appengine.api import search 40 | 41 | 42 | def reinitAll(sample_data=True): 43 | """ 44 | Deletes all product entities and documents, essentially resetting the app 45 | state, then loads in static sample data if requested. Hardwired for the 46 | expected product types in the sample data. 47 | (Re)loads store location data from stores.py as well. 48 | This function is intended to be run 'offline' (e.g., via a Task Queue task). 49 | As an extension to this functionality, the channel ID could be used to notify 50 | when done.""" 51 | 52 | # delete all the product and review entities 53 | review_keys = models.Review.query().fetch(keys_only=True) 54 | ndb.delete_multi(review_keys) 55 | prod_keys = models.Product.query().fetch(keys_only=True) 56 | ndb.delete_multi(prod_keys) 57 | # delete all the associated product documents in the doc and 58 | # store indexes 59 | docs.Product.deleteAllInProductIndex() 60 | docs.Store.deleteAllInIndex() 61 | # load in sample data if indicated 62 | if sample_data: 63 | logging.info('Loading product sample data') 64 | # Load from csv sample files. 65 | # The following are hardwired to the format of the sample data files 66 | # for the two example product types ('books' and 'hd televisions')-- see 67 | # categories.py 68 | datafile = os.path.join('data', config.SAMPLE_DATA_BOOKS) 69 | # books 70 | reader = csv.DictReader( 71 | open(datafile, 'r'), 72 | ['pid', 'name', 'category', 'price', 73 | 'publisher', 'title', 'pages', 'author', 74 | 'description', 'isbn']) 75 | importData(reader) 76 | datafile = os.path.join('data', config.SAMPLE_DATA_TVS) 77 | # tvs 78 | reader = csv.DictReader( 79 | open(datafile, 'r'), 80 | ['pid', 'name', 'category', 'price', 81 | 'size', 'brand', 'tv_type', 82 | 'description']) 83 | importData(reader) 84 | 85 | # next create docs from store location info 86 | loadStoreLocationData() 87 | 88 | logging.info('Re-initialization complete.') 89 | 90 | def loadStoreLocationData(): 91 | # create documents from store location info 92 | # currently logs but otherwise swallows search errors. 93 | slocs = stores.stores 94 | for s in slocs: 95 | logging.info("s: %s", s) 96 | geopoint = search.GeoPoint(s[3][0], s[3][1]) 97 | fields = [search.TextField(name=docs.Store.STORE_NAME, value=s[1]), 98 | search.TextField(name=docs.Store.STORE_ADDRESS, value=s[2]), 99 | search.GeoField(name=docs.Store.STORE_LOCATION, value=geopoint) 100 | ] 101 | d = search.Document(doc_id=s[0], fields=fields) 102 | try: 103 | add_result = search.Index(config.STORE_INDEX_NAME).put(d) 104 | except search.Error: 105 | logging.exception("Error adding document:") 106 | 107 | 108 | def importData(reader): 109 | """Import via the csv reader iterator using the specified batch size as set in 110 | the config file. We want to ensure the batch is not too large-- we allow 100 111 | rows/products max per batch.""" 112 | MAX_BATCH_SIZE = 100 113 | rows = [] 114 | # index in batches 115 | # ensure the batch size in the config file is not over the max or < 1. 116 | batchsize = utils.intClamp(config.IMPORT_BATCH_SIZE, 1, MAX_BATCH_SIZE) 117 | logging.debug('batchsize: %s', batchsize) 118 | for row in reader: 119 | if len(rows) == batchsize: 120 | docs.Product.buildProductBatch(rows) 121 | rows = [row] 122 | else: 123 | rows.append(row) 124 | if rows: 125 | docs.Product.buildProductBatch(rows) 126 | 127 | 128 | class AdminHandler(BaseHandler): 129 | """Displays the admin page.""" 130 | 131 | def buildAdminPage(self, notification=None): 132 | # If necessary, build the app's product categories now. This is done only 133 | # if there are no Category entities in the datastore. 134 | models.Category.buildAllCategories() 135 | tdict = { 136 | 'sampleb': config.SAMPLE_DATA_BOOKS, 137 | 'samplet': config.SAMPLE_DATA_TVS, 138 | 'update_sample': config.DEMO_UPDATE_BOOKS_DATA} 139 | if notification: 140 | tdict['notification'] = notification 141 | self.render_template('admin.html', tdict) 142 | 143 | @BaseHandler.logged_in 144 | def get(self): 145 | action = self.request.get('action') 146 | if action == 'reinit': 147 | # reinitialise the app data to the sample data 148 | defer(reinitAll) 149 | self.buildAdminPage(notification="Reinitialization performed.") 150 | elif action == 'demo_update': 151 | # update the sample data, from (hardwired) book update 152 | # data. Demonstrates updating some existing products, and adding some new 153 | # ones. 154 | logging.info('Loading product sample update data') 155 | # The following is hardwired to the known format of the sample data file 156 | datafile = os.path.join('data', config.DEMO_UPDATE_BOOKS_DATA) 157 | reader = csv.DictReader( 158 | open(datafile, 'r'), 159 | ['pid', 'name', 'category', 'price', 160 | 'publisher', 'title', 'pages', 'author', 161 | 'description', 'isbn']) 162 | for row in reader: 163 | docs.Product.buildProduct(row) 164 | self.buildAdminPage(notification="Demo update performed.") 165 | 166 | elif action == 'update_ratings': 167 | self.update_ratings() 168 | self.buildAdminPage(notification="Ratings update performed.") 169 | else: 170 | self.buildAdminPage() 171 | 172 | def update_ratings(self): 173 | """Find the products that have had an average ratings change, and need their 174 | associated documents updated (re-indexed) to reflect that change; and 175 | re-index those docs in batch. There will only 176 | be such products if config.BATCH_RATINGS_UPDATE is True; otherwise the 177 | associated documents will be updated right away.""" 178 | # get the pids of the products that need review info updated in their 179 | # associated documents. 180 | pkeys = models.Product.query( 181 | models.Product.needs_review_reindex == True).fetch(keys_only=True) 182 | # re-index these docs in batch 183 | models.Product.updateProdDocsWithNewRating(pkeys) 184 | 185 | 186 | class DeleteProductHandler(BaseHandler): 187 | """Remove data for the product with the given pid, including that product's 188 | reviews and its associated indexed document.""" 189 | 190 | @BaseHandler.logged_in 191 | def post(self): 192 | pid = self.request.get('pid') 193 | if not pid: # this should not be reached 194 | msg = 'There was a problem: no product id given.' 195 | logging.error(msg) 196 | url = '/' 197 | linktext = 'Go to product search page.' 198 | self.render_template( 199 | 'notification.html', 200 | {'title': 'Error', 'msg': msg, 201 | 'goto_url': url, 'linktext': linktext}) 202 | return 203 | 204 | # Delete the product entity within a transaction, and define transactional 205 | # tasks for deleting the product's reviews and its associated document. 206 | # These tasks will only be run if the transaction successfully commits. 207 | def _tx(): 208 | prod = models.Product.get_by_id(pid) 209 | if prod: 210 | prod.key.delete() 211 | defer(models.Review.deleteReviews, prod.key.id(), _transactional=True) 212 | defer( 213 | docs.Product.removeProductDocByPid, 214 | prod.key.id(), _transactional=True) 215 | 216 | ndb.transaction(_tx) 217 | # indicate success 218 | msg = ( 219 | 'The product with product id %s has been ' + 220 | 'successfully removed.') % (pid,) 221 | url = '/' 222 | linktext = 'Go to product search page.' 223 | self.render_template( 224 | 'notification.html', 225 | {'title': 'Product Removed', 'msg': msg, 226 | 'goto_url': url, 'linktext': linktext}) 227 | 228 | 229 | class CreateProductHandler(BaseHandler): 230 | """Handler to create a new product: this constitutes both a product entity 231 | and its associated indexed document.""" 232 | 233 | def parseParams(self): 234 | """Filter the param set to the expected params.""" 235 | 236 | pid = self.request.get('pid') 237 | doc = docs.Product.getDocFromPid(pid) 238 | params = {} 239 | if doc: # populate default params from the doc 240 | fields = doc.fields 241 | for f in fields: 242 | params[f.name] = f.value 243 | else: 244 | # start with the 'core' fields 245 | params = { 246 | 'pid': uuid.uuid4().hex, # auto-generate default UID 247 | 'name': '', 248 | 'description': '', 249 | 'category': '', 250 | 'price': ''} 251 | pf = categories.product_dict 252 | # add the fields specific to the categories 253 | for _, cdict in pf.iteritems(): 254 | temp = {} 255 | for elt in cdict.keys(): 256 | temp[elt] = '' 257 | params.update(temp) 258 | 259 | for k, v in params.iteritems(): 260 | # Process the request params. Possibly replace default values. 261 | params[k] = self.request.get(k, v) 262 | return params 263 | 264 | @BaseHandler.logged_in 265 | def get(self): 266 | params = self.parseParams() 267 | self.render_template('create_product.html', params) 268 | 269 | @BaseHandler.logged_in 270 | def post(self): 271 | self.createProduct(self.parseParams()) 272 | 273 | def createProduct(self, params): 274 | """Create a product entity and associated document from the given params 275 | dict.""" 276 | 277 | try: 278 | product = docs.Product.buildProduct(params) 279 | self.redirect( 280 | '/product?' + urllib.urlencode( 281 | {'pid': product.pid, 'pname': params['name'], 282 | 'category': product.category 283 | })) 284 | except errors.Error as e: 285 | logging.exception('Error:') 286 | params['error_message'] = e.error_message 287 | self.render_template('create_product.html', params) 288 | 289 | 290 | -------------------------------------------------------------------------------- /product_search_python/app.yaml: -------------------------------------------------------------------------------- 1 | application: your-app-id 2 | version: one 3 | runtime: python27 4 | threadsafe: true 5 | api_version: 1 6 | 7 | handlers: 8 | - url: /static 9 | static_dir: static 10 | 11 | - url: /admin/.* 12 | script: admin.application 13 | login: required 14 | secure: always 15 | 16 | - url: .* 17 | script: main.application 18 | 19 | builtins: 20 | - deferred: on 21 | # - appstats: on 22 | 23 | libraries: 24 | - name: jinja2 25 | version: "2.6" 26 | 27 | inbound_services: 28 | - warmup 29 | -------------------------------------------------------------------------------- /product_search_python/base_handler.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright 2012 Google Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | """ The base request handler class. 18 | """ 19 | 20 | 21 | import webapp2 22 | from webapp2_extras import jinja2 23 | import json 24 | 25 | from google.appengine.api import users 26 | 27 | 28 | class BaseHandler(webapp2.RequestHandler): 29 | """The other handlers inherit from this class. Provides some helper methods 30 | for rendering a template and generating template links.""" 31 | 32 | @classmethod 33 | def logged_in(cls, handler_method): 34 | """ 35 | This decorator requires a logged-in user, and returns 403 otherwise. 36 | """ 37 | def auth_required(self, *args, **kwargs): 38 | if (users.get_current_user() or 39 | self.request.headers.get('X-AppEngine-Cron')): 40 | handler_method(self, *args, **kwargs) 41 | else: 42 | self.error(403) 43 | return auth_required 44 | 45 | @webapp2.cached_property 46 | def jinja2(self): 47 | return jinja2.get_jinja2(app=self.app) 48 | 49 | def render_template(self, filename, template_args): 50 | template_args.update(self.generateSidebarLinksDict()) 51 | self.response.write(self.jinja2.render_template(filename, **template_args)) 52 | 53 | def render_json(self, response): 54 | self.response.write("%s(%s);" % (self.request.GET['callback'], 55 | json.dumps(response))) 56 | 57 | def getLoginLink(self): 58 | """Generate login or logout link and text, depending upon the logged-in 59 | status of the client.""" 60 | if users.get_current_user(): 61 | url = users.create_logout_url(self.request.uri) 62 | url_linktext = 'Logout' 63 | else: 64 | url = users.create_login_url(self.request.uri) 65 | url_linktext = 'Login' 66 | return (url, url_linktext) 67 | 68 | def getAdminManageLink(self): 69 | """Build link to the admin management page, if the user is logged in.""" 70 | if users.get_current_user(): 71 | admin_url = '/admin/manage' 72 | return (admin_url, 'Admin/Add sample data') 73 | else: 74 | return (None, None) 75 | 76 | def createProductAdminLink(self): 77 | if users.get_current_user(): 78 | admin_create_url = '/admin/create_product' 79 | return (admin_create_url, 'Create new product (admin)') 80 | else: 81 | return (None, None) 82 | 83 | def generateSidebarLinksDict(self): 84 | """Build a dict containing login/logout and admin links, which will be 85 | included in the sidebar for all app pages.""" 86 | 87 | url, url_linktext = self.getLoginLink() 88 | admin_create_url, admin_create_text = self.createProductAdminLink() 89 | admin_url, admin_text = self.getAdminManageLink() 90 | return { 91 | 'admin_create_url': admin_create_url, 92 | 'admin_create_text': admin_create_text, 93 | 'admin_url': admin_url, 94 | 'admin_text': admin_text, 95 | 'url': url, 96 | 'url_linktext': url_linktext 97 | } 98 | 99 | -------------------------------------------------------------------------------- /product_search_python/categories.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright 2012 Google Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | """Specifies product category information for the app. In this sample, there 18 | are two categories: books, and televisions. 19 | """ 20 | from google.appengine.api import search 21 | 22 | 23 | televisions = {'name': 'hd televisions', 'children': []} 24 | books = {'name': 'books', 'children': []} 25 | 26 | ctree = {'name': 'root', 'children': [books, televisions]} 27 | 28 | # [The core fields that all products share are: product id, name, description, 29 | # category, category name, and price] 30 | # Define the non-'core' (differing) product fields for each category 31 | # above, and their types. 32 | product_dict = {'hd televisions': {'size': search.NumberField, 33 | 'brand': search.TextField, 34 | 'tv_type': search.TextField}, 35 | 'books': {'publisher': search.TextField, 36 | 'pages': search.NumberField, 37 | 'author': search.TextField, 38 | 'title': search.TextField, 39 | 'isbn': search.TextField} 40 | } 41 | -------------------------------------------------------------------------------- /product_search_python/config.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright 2012 Google Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | """ Holds configuration settings. 18 | """ 19 | 20 | 21 | PRODUCT_INDEX_NAME = 'productsearch1' # The document index name. 22 | # An index name must be a visible printable 23 | # ASCII string not starting with '!'. Whitespace characters are 24 | # excluded. 25 | 26 | STORE_INDEX_NAME = 'stores1' 27 | 28 | # set BATCH_RATINGS_UPDATE to False to update documents with changed ratings 29 | # info right away. If True, updates will only occur when triggered by 30 | # an admin request or a cron job. See cron.yaml for an example. 31 | BATCH_RATINGS_UPDATE = False 32 | # BATCH_RATINGS_UPDATE = True 33 | 34 | # The max and min (integer) ratings values allowed. 35 | RATING_MIN = 1 36 | RATING_MAX = 5 37 | 38 | # the number of search results to display per page 39 | DOC_LIMIT = 3 40 | 41 | SAMPLE_DATA_BOOKS = 'sample_data_books.csv' 42 | SAMPLE_DATA_TVS = 'sample_data_tvs.csv' 43 | DEMO_UPDATE_BOOKS_DATA = 'sample_data_books_update.csv' 44 | 45 | # the size of the import batches, when reading from the csv file. Must not 46 | # exceed 100. 47 | IMPORT_BATCH_SIZE = 5 48 | -------------------------------------------------------------------------------- /product_search_python/cron.yaml: -------------------------------------------------------------------------------- 1 | cron: 2 | - description: reindex any documents that need ratings update due to new reviews 3 | url: '/admin/update_ratings_info' 4 | schedule: every 15 minutes -------------------------------------------------------------------------------- /product_search_python/data/sample_data_books.csv: -------------------------------------------------------------------------------- 1 | testbookprod0,The Epicure's Lament,books,12.24,Anchor,The Epicure's Lament,368,Kate Christensen,"Christensen's two previous novels (Jeremy Thrane; In the Drink) were delightfully believable, sympathetic contemporary narratives filled with wry humor and appealing protagonists. Here she ups the ante, with loftier literary aspirations and succeeds masterfully.",038572098X 2 | 3 | testbookprod1,Otherwise Known as the Human Condition,books,12.24,Graywolf Press,Otherwise Known as the Human Condition: Selected Essays and Reviews,432,Geoff Dyer,"Starred Review. In this new collection of previously published writings, Dyer (Jeff in Venice, Death in Varanasi) traverses a broad territory stretching from photographers such as Richard Avedon and William Gedney (His gaze is neither penetrating nor alert but, on reflection, we would amend that verdict to accepting); musicians Miles Davis and Def Leppard; writers like D.H. Lawrence, Ian McEwan, and Richard Ford; as well as personal ruminations on, say, reader's block.",1555975798 4 | 5 | testbookprod2,Case Histories: A Novel,books,7.99,"Little, Brown and Company",Case Histories: A Novel,400,Kate Atkinson,"In this ambitious fourth novel from Whitbread winner Atkinson (Behind the Scenes at the Museum), private detective Jackson Brodie-- ex-cop, ex-husband and weekend dad-- takes on three cases involving past crimes that occurred in and around London. The first case introduces two middle-aged sisters who, after the death of their vile, distant father, look again into the disappearance of their beloved sister Olivia, last seen at three years old, while they were camping under the stars during an oppressive heat wave. A retired lawyer who lives only on the fumes of possible justice next enlists Jackson's aid in solving the brutal killing of his grown daughter 10 years earlier. In the third dog-eared case file, the sibling of an infamous ax-bludgeoner seeks a reunion with her niece, who as a baby was a witness to murder. Jackson's reluctant persistence heats up these cold cases and by happenstance leads him to reassess his own painful history. The humility of the extraordinary, unabashed characters is skillfully revealed with humor and surprise. Atkinson contrasts the inevitable results of family dysfunction with random fate, gracefully weaving the three stories into a denouement that taps into collective wishful thinking and suggests that warmth and safety may be found in the aftermath of blood and abandonment. Atkinson's meaty, satisfying prose will attract many eager readers.",0316033480 6 | 7 | testbookprod3,Distrust That Particular Flavor,books,16.43,Penguin Publishing,Distrust That Particular Flavor ,935,William Gibson,"William Gibson is known primarily as a novelist, with his work ranging from his groundbreaking first novel, Neuromancer, to his more recent contemporary bestsellers Pattern Recognition, Spook Country, and Zero History. During those nearly thirty years, though, Gibson has been sought out by widely varying publications for his insights into contemporary culture. Wired magazine sent him to Singapore to report on one of the world's most buttoned-up states. The New York Times Magazine asked him to describe what was wrong with the Internet. Rolling Stone published his essay on the ways our lives are all 'soundtracked' by the music and the culture around us. And in a speech at the 2010 Book Expo, he memorably described the interactive relationship between writer and reader. 8 | 9 | These essays and articles have never been collected-until now. Some have never appeared in print at all. In addition, Distrust That Particular Flavor includes journalism from small publishers, online sources, and magazines no longer in existence. This volume will be essential reading for any lover of William Gibson's novels. Distrust That Particular Flavor offers readers a privileged view into the mind of a writer whose thinking has shaped not only a generation of writers but our entire culture.",039915843X 10 | 11 | testbookprod4,"Thinking, Fast and Slow",books,15.00,"Farrar, Straus and Giroux","Thinking, Fast and Slow",512,Daniel Kahneman,"Drawing on decades of research in psychology that resulted in a Nobel Prize in Economic Sciences, Daniel Kahneman takes readers on an exploration of what influences thought example by example, sometimes with unlikely word pairs like 'vomit' and 'banana'. System 1 and System 2, the fast and slow types of thinking, become characters that illustrate the psychology behind things we think we understand but really don't, such as intuition. Kahneman's transparent and careful treatment of his subject has the potential to change how we think, not just about thinking, but about how we live our lives. Thinking, Fast and Slow gives deep--and sometimes frightening--insight about what goes on inside our heads: the psychological basis for reactions, judgments, recognition, choices, conclusions, and much more.",0374275637 12 | 13 | testbookprod5,The Big Short,books,9.93,W. W. Norton & Company,The Big Short: Inside the Doomsday Machine,320,Michael Lewis,"The real story of the crash began in bizarre feeder markets where the sun doesn't shine and the SEC doesn't dare, or bother, to tread: the bond and real estate derivative markets where geeks invent impenetrable securities to profit from the misery of lower--and middle--class Americans who can't pay their debts. The smart people who understood what was or might be happening were paralyzed by hope and fear; in any case, they weren't talking. 14 | 15 | Michael Lewis creates a fresh, character-driven narrative brimming with indignation and dark humor, a fitting sequel to his #1 bestseller Liar's Poker. Out of a handful of unlikely--really unlikely--heroes, Lewis fashions a story as compelling and unusual as any of his earlier bestsellers, proving yet again that he is the finest and funniest chronicler of our time.",0393338827 16 | 17 | testbookprod6,Composed: A Memoir,books,25.60,Penguin,Composed: A Memoir,206,Rosanne Cash,"As moving, disarming, and elusive as one of her classic songs, Composed is Rosanne Cash's testament to the power of art, tradition, and love to transform a life. For more than three decades she has been one of the most compelling figures in popular music, having moved gracefully from Nashville stardom to critical recognition as a singer-songwriter and author of essays and short stories. Her remarkable body of work has often been noted for its emotional acuity, its rich and resonant imagery, and its unsparing honesty. Those qualities have enabled her to establish a unique intimacy with her audiences, and it is those qualities that inform her long-awaited memoir.",0143119397 18 | 19 | testbookprod7,Runaway,books,11.99,Vintage,Runaway,352,Alice Munro,"Alice Munro's bestselling and rapturously acclaimed Runaway is a book of extraordinary stories about love and its infinite betrayals and surprises, from the title story about a young woman who, though she thinks she wants to, is incapable of leaving her husband, to three stories about a woman named Juliet and the emotions that complicate the luster of her intimate relationships. In Munro's hands, the people she writes about-- women of all ages and circumstances, and their friends, lovers, parents, and children-- become as vivid as our own neighbors. It is her miraculous gift to make these stories as real and unforgettable as our own.",1400077915 20 | -------------------------------------------------------------------------------- /product_search_python/data/sample_data_books_update.csv: -------------------------------------------------------------------------------- 1 | testbookprod7,Runaway,books,10.99,Vintage,Runaway,352,Alice Munro,"The incomparable Alice Munro's bestselling and rapturously acclaimed Runaway is a book of extraordinary stories about love and its infinite betrayals and surprises, from the title story about a young woman who, though she thinks she wants to, is incapable of leaving her husband, to three stories about a woman named Juliet and the emotions that complicate the luster of her intimate relationships. In Munro's hands, the people she writes about-- women of all ages and circumstances, and their friends, lovers, parents, and children-- become as vivid as our own neighbors. It is her miraculous gift to make these stories as real and unforgettable as our own.",1400077915 2 | 3 | testbookprod8,The adventures of Sherlock Holmes,books,20.08,Baker Books,The adventures of Sherlock Holmes,208,Sir Arthur Conan Doyle,The adventures of Sherlock Holmes,123458 4 | testbookprod9,The adventures of Sherlock Holmes II,books,20.09,Baker Books,The adventures of Sherlock Holmes II,209,Sir Arthur Conan Doyle,The adventures of Sherlock Holmes,123459 -------------------------------------------------------------------------------- /product_search_python/data/sample_data_tvs.csv: -------------------------------------------------------------------------------- 1 | testtvprod0,tv0,hd televisions,1999.99,32,Mega TVs,plasma,Mega TV Plasma 32" 2 | testtvprod1,tv1,hd televisions,1299.99,28,Mega TVs,lcd,Mega TV LCD 28" 3 | testtvprod2,tv2,hd televisions,899.99,26,Mega TVs,plasma,Mega TV Plasma 26" 4 | testtvprod3,tv3,hd televisions,1899.99,32,Mega TVs,lcd,Mega TV LCD 32" 5 | -------------------------------------------------------------------------------- /product_search_python/docs.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright 2012 Google Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | """ Contains 'helper' classes for managing search.Documents. 18 | BaseDocumentManager provides some common utilities, and the Product subclass 19 | adds some Product-document-specific helper methods. 20 | """ 21 | 22 | import collections 23 | import copy 24 | import datetime 25 | import logging 26 | import re 27 | import string 28 | import urllib 29 | 30 | import categories 31 | import config 32 | import errors 33 | import models 34 | 35 | from google.appengine.api import search 36 | from google.appengine.ext import ndb 37 | 38 | 39 | class BaseDocumentManager(object): 40 | """Abstract class. Provides helper methods to manage search.Documents.""" 41 | 42 | _INDEX_NAME = None 43 | _VISIBLE_PRINTABLE_ASCII = frozenset( 44 | set(string.printable) - set(string.whitespace)) 45 | 46 | def __init__(self, doc): 47 | """Builds a dict of the fields mapped against the field names, for 48 | efficient access. 49 | """ 50 | self.doc = doc 51 | fields = doc.fields 52 | 53 | def getFieldVal(self, fname): 54 | """Get the value of the document field with the given name. If there is 55 | more than one such field, the method returns None.""" 56 | try: 57 | return self.doc.field(fname).value 58 | except ValueError: 59 | return None 60 | 61 | def setFirstField(self, new_field): 62 | """Set the value of the (first) document field with the given name.""" 63 | for i, field in enumerate(self.doc.fields): 64 | if field.name == new_field.name: 65 | self.doc.fields[i] = new_field 66 | return True 67 | return False 68 | 69 | @classmethod 70 | def isValidDocId(cls, doc_id): 71 | """Checks if the given id is a visible printable ASCII string not starting 72 | with '!'. Whitespace characters are excluded. 73 | """ 74 | for char in doc_id: 75 | if char not in cls._VISIBLE_PRINTABLE_ASCII: 76 | return False 77 | return not doc_id.startswith('!') 78 | 79 | @classmethod 80 | def getIndex(cls): 81 | return search.Index(name=cls._INDEX_NAME) 82 | 83 | @classmethod 84 | def deleteAllInIndex(cls): 85 | """Delete all the docs in the given index.""" 86 | docindex = cls.getIndex() 87 | 88 | try: 89 | while True: 90 | # until no more documents, get a list of documents, 91 | # constraining the returned objects to contain only the doc ids, 92 | # extract the doc ids, and delete the docs. 93 | document_ids = [document.doc_id 94 | for document in docindex.get_range(ids_only=True)] 95 | if not document_ids: 96 | break 97 | docindex.delete(document_ids) 98 | except search.Error: 99 | logging.exception("Error removing documents:") 100 | 101 | @classmethod 102 | def getDoc(cls, doc_id): 103 | """Return the document with the given doc id. One way to do this is via 104 | the get_range method, as shown here. If the doc id is not in the 105 | index, the first doc in the index will be returned instead, so we need 106 | to check for that case.""" 107 | if not doc_id: 108 | return None 109 | try: 110 | index = cls.getIndex() 111 | response = index.get_range( 112 | start_id=doc_id, limit=1, include_start_object=True) 113 | if response.results and response.results[0].doc_id == doc_id: 114 | return response.results[0] 115 | return None 116 | except search.InvalidRequest: # catches ill-formed doc ids 117 | return None 118 | 119 | @classmethod 120 | def removeDocById(cls, doc_id): 121 | """Remove the doc with the given doc id.""" 122 | try: 123 | cls.getIndex().delete(doc_id) 124 | except search.Error: 125 | logging.exception("Error removing doc id %s.", doc_id) 126 | 127 | @classmethod 128 | def add(cls, documents): 129 | """wrapper for search index add method; specifies the index name.""" 130 | try: 131 | return cls.getIndex().put(documents) 132 | except search.Error: 133 | logging.exception("Error adding documents.") 134 | 135 | 136 | class Store(BaseDocumentManager): 137 | 138 | _INDEX_NAME = config.STORE_INDEX_NAME 139 | STORE_NAME = 'store_name' 140 | STORE_LOCATION = 'store_location' 141 | STORE_ADDRESS = 'store_address' 142 | 143 | 144 | class Product(BaseDocumentManager): 145 | """Provides helper methods to manage Product documents. All Product documents 146 | built using these methods will include a core set of fields (see the 147 | _buildCoreProductFields method). We use the given product id (the Product 148 | entity key) as the doc_id. This is not required for the entity/document 149 | design-- each explicitly point to each other, allowing their ids to be 150 | decoupled-- but using the product id as the doc id allows a document to be 151 | reindexed given its product info, without having to fetch the 152 | existing document.""" 153 | 154 | _INDEX_NAME = config.PRODUCT_INDEX_NAME 155 | 156 | # 'core' product document field names 157 | PID = 'pid' 158 | DESCRIPTION = 'description' 159 | CATEGORY = 'category' 160 | PRODUCT_NAME = 'name' 161 | PRICE = 'price' 162 | AVG_RATING = 'ar' #average rating 163 | UPDATED = 'modified' 164 | 165 | _SORT_OPTIONS = [ 166 | [AVG_RATING, 'average rating', search.SortExpression( 167 | expression=AVG_RATING, 168 | direction=search.SortExpression.DESCENDING, default_value=0)], 169 | [PRICE, 'price', search.SortExpression( 170 | # other examples: 171 | # expression='max(price, 14.99)' 172 | # If you access _score in your sort expressions, 173 | # your SortOptions should include a scorer. 174 | # e.g. search.SortOptions(match_scorer=search.MatchScorer(),...) 175 | # Then, you can access the score to build expressions like: 176 | # expression='price * _score' 177 | expression=PRICE, 178 | direction=search.SortExpression.ASCENDING, default_value=9999)], 179 | [UPDATED, 'modified', search.SortExpression( 180 | expression=UPDATED, 181 | direction=search.SortExpression.DESCENDING, default_value=1)], 182 | [CATEGORY, 'category', search.SortExpression( 183 | expression=CATEGORY, 184 | direction=search.SortExpression.ASCENDING, default_value='')], 185 | [PRODUCT_NAME, 'product name', search.SortExpression( 186 | expression=PRODUCT_NAME, 187 | direction=search.SortExpression.ASCENDING, default_value='zzz')] 188 | ] 189 | 190 | _SORT_MENU = None 191 | _SORT_DICT = None 192 | 193 | 194 | @classmethod 195 | def deleteAllInProductIndex(cls): 196 | cls.deleteAllInIndex() 197 | 198 | @classmethod 199 | def getSortMenu(cls): 200 | if not cls._SORT_MENU: 201 | cls._buildSortMenu() 202 | return cls._SORT_MENU 203 | 204 | @classmethod 205 | def getSortDict(cls): 206 | if not cls._SORT_DICT: 207 | cls._buildSortDict() 208 | return cls._SORT_DICT 209 | 210 | @classmethod 211 | def _buildSortMenu(cls): 212 | """Build the default set of sort options used for Product search. 213 | Of these options, all but 'relevance' reference core fields that 214 | all Products will have.""" 215 | res = [(elt[0], elt[1]) for elt in cls._SORT_OPTIONS] 216 | cls._SORT_MENU = [('relevance', 'relevance')] + res 217 | 218 | @classmethod 219 | def _buildSortDict(cls): 220 | """Build a dict that maps sort option keywords to their corresponding 221 | SortExpressions.""" 222 | cls._SORT_DICT = {} 223 | for elt in cls._SORT_OPTIONS: 224 | cls._SORT_DICT[elt[0]] = elt[2] 225 | 226 | @classmethod 227 | def getDocFromPid(cls, pid): 228 | """Given a pid, get its doc. We're using the pid as the doc id, so we can 229 | do this via a direct fetch.""" 230 | return cls.getDoc(pid) 231 | 232 | @classmethod 233 | def removeProductDocByPid(cls, pid): 234 | """Given a doc's pid, remove the doc matching it from the product 235 | index.""" 236 | cls.removeDocById(pid) 237 | 238 | @classmethod 239 | def updateRatingInDoc(cls, doc_id, avg_rating): 240 | # get the associated doc from the doc id in the product entity 241 | doc = cls.getDoc(doc_id) 242 | if doc: 243 | pdoc = cls(doc) 244 | pdoc.setAvgRating(avg_rating) 245 | # The use of the same id will cause the existing doc to be reindexed. 246 | return doc 247 | else: 248 | raise errors.OperationFailedError( 249 | 'Could not retrieve doc associated with id %s' % (doc_id,)) 250 | 251 | @classmethod 252 | def updateRatingsInfo(cls, doc_id, avg_rating): 253 | """Given a models.Product entity, update and reindex the associated 254 | document with the product entity's current average rating. """ 255 | 256 | ndoc = cls.updateRatingInDoc(doc_id, avg_rating) 257 | # reindex the returned updated doc 258 | return cls.add(ndoc) 259 | 260 | # 'accessor' convenience methods 261 | 262 | def getPID(self): 263 | """Get the value of the 'pid' field of a Product doc.""" 264 | return self.getFieldVal(self.PID) 265 | 266 | def getName(self): 267 | """Get the value of the 'name' field of a Product doc.""" 268 | return self.getFieldVal(self.PRODUCT_NAME) 269 | 270 | def getDescription(self): 271 | """Get the value of the 'description' field of a Product doc.""" 272 | return self.getFieldVal(self.DESCRIPTION) 273 | 274 | def getCategory(self): 275 | """Get the value of the 'cat' field of a Product doc.""" 276 | return self.getFieldVal(self.CATEGORY) 277 | 278 | def setCategory(self, cat): 279 | """Set the value of the 'cat' (category) field of a Product doc.""" 280 | return self.setFirstField(search.NumberField(name=self.CATEGORY, value=cat)) 281 | 282 | def getAvgRating(self): 283 | """Get the value of the 'ar' (average rating) field of a Product doc.""" 284 | return self.getFieldVal(self.AVG_RATING) 285 | 286 | def setAvgRating(self, ar): 287 | """Set the value of the 'ar' field of a Product doc.""" 288 | return self.setFirstField(search.NumberField(name=self.AVG_RATING, value=ar)) 289 | 290 | def getPrice(self): 291 | """Get the value of the 'price' field of a Product doc.""" 292 | return self.getFieldVal(self.PRICE) 293 | 294 | @classmethod 295 | def generateRatingsBuckets(cls, query_string): 296 | """Builds a dict of ratings 'buckets' and their counts, based on the 297 | value of the 'avg_rating" field for the documents retrieved by the given 298 | query. See the 'generateRatingsLinks' method. This information will 299 | be used to generate sidebar links that allow the user to drill down in query 300 | results based on rating. 301 | 302 | For demonstration purposes only; this will be expensive for large data 303 | sets. 304 | """ 305 | 306 | # do the query on the *full* search results 307 | # to generate the facet information, imitating what may in future be 308 | # provided by the FTS API. 309 | try: 310 | sq = search.Query( 311 | query_string=query_string.strip()) 312 | search_results = cls.getIndex().search(sq) 313 | except search.Error: 314 | logging.exception('An error occurred on search.') 315 | return None 316 | 317 | ratings_buckets = collections.defaultdict(int) 318 | # populate the buckets 319 | for res in search_results: 320 | ratings_buckets[int((cls(res)).getAvgRating() or 0)] += 1 321 | return ratings_buckets 322 | 323 | @classmethod 324 | def generateRatingsLinks(cls, query, phash): 325 | """Given a dict of ratings 'buckets' and their counts, 326 | builds a list of html snippets, to be displayed in the sidebar when 327 | showing results of a query. Each is a link that runs the query, additionally 328 | filtered by the indicated ratings interval.""" 329 | 330 | ratings_buckets = cls.generateRatingsBuckets(query) 331 | if not ratings_buckets: 332 | return None 333 | rlist = [] 334 | for k in range(config.RATING_MIN, config.RATING_MAX+1): 335 | try: 336 | v = ratings_buckets[k] 337 | except KeyError: 338 | return 339 | # build html 340 | if k < 5: 341 | htext = '%s-%s (%s)' % (k, k+1, v) 342 | else: 343 | htext = '%s (%s)' % (k, v) 344 | phash['rating'] = k 345 | hlink = '/psearch?' + urllib.urlencode(phash) 346 | rlist.append((hlink, htext)) 347 | return rlist 348 | 349 | @classmethod 350 | def _buildCoreProductFields( 351 | cls, pid, name, description, category, category_name, price): 352 | """Construct a 'core' document field list for the fields common to all 353 | Products. The various categories (as defined in the file 'categories.py'), 354 | may add additional specialized fields; these will be appended to this 355 | core list. (see _buildProductFields).""" 356 | fields = [search.TextField(name=cls.PID, value=pid), 357 | # The 'updated' field is always set to the current date. 358 | search.DateField(name=cls.UPDATED, 359 | value=datetime.datetime.now().date()), 360 | search.TextField(name=cls.PRODUCT_NAME, value=name), 361 | # strip the markup from the description value, which can 362 | # potentially come from user input. We do this so that 363 | # we don't need to sanitize the description in the 364 | # templates, showing off the Search API's ability to mark up query 365 | # terms in generated snippets. This is done only for 366 | # demonstration purposes; in an actual app, 367 | # it would be preferrable to use a library like Beautiful Soup 368 | # instead. 369 | # We'll let the templating library escape all other rendered 370 | # values for us, so this is the only field we do this for. 371 | search.TextField( 372 | name=cls.DESCRIPTION, 373 | value=re.sub(r'<[^>]*?>', '', description)), 374 | search.AtomField(name=cls.CATEGORY, value=category), 375 | search.NumberField(name=cls.AVG_RATING, value=0.0), 376 | search.NumberField(name=cls.PRICE, value=price) 377 | ] 378 | return fields 379 | 380 | @classmethod 381 | def _buildProductFields(cls, pid=None, category=None, name=None, 382 | description=None, category_name=None, price=None, **params): 383 | """Build all the additional non-core fields for a document of the given 384 | product type (category), using the given params dict, and the 385 | already-constructed list of 'core' fields. All such additional 386 | category-specific fields are treated as required. 387 | """ 388 | 389 | fields = cls._buildCoreProductFields( 390 | pid, name, description, category, category_name, price) 391 | # get the specification of additional (non-'core') fields for this category 392 | pdict = categories.product_dict.get(category_name) 393 | if pdict: 394 | # for all fields 395 | for k, field_type in pdict.iteritems(): 396 | # see if there is a value in the given params for that field. 397 | # if there is, get the field type, create the field, and append to the 398 | # document field list. 399 | if k in params: 400 | v = params[k] 401 | if field_type == search.NumberField: 402 | try: 403 | val = float(v) 404 | fields.append(search.NumberField(name=k, value=val)) 405 | except ValueError: 406 | error_message = ('bad value %s for field %s of type %s' % 407 | (k, v, field_type)) 408 | logging.error(error_message) 409 | raise errors.OperationFailedError(error_message) 410 | elif field_type == search.TextField: 411 | fields.append(search.TextField(name=k, value=str(v))) 412 | else: 413 | # you may want to add handling of other field types for generality. 414 | # Not needed for our current sample data. 415 | logging.warn('not processed: %s, %s, of type %s', k, v, field_type) 416 | else: 417 | error_message = ('value not given for field "%s" of field type "%s"' 418 | % (k, field_type)) 419 | logging.warn(error_message) 420 | raise errors.OperationFailedError(error_message) 421 | else: 422 | # else, did not have an entry in the params dict for the given field. 423 | logging.warn( 424 | 'product field information not found for category name %s', 425 | params['category_name']) 426 | return fields 427 | 428 | @classmethod 429 | def _createDocument( 430 | cls, pid=None, category=None, name=None, description=None, 431 | category_name=None, price=None, **params): 432 | """Create a Document object from given params.""" 433 | # check for the fields that are always required. 434 | if pid and category and name: 435 | # First, check that the given pid has only visible ascii characters, 436 | # and does not contain whitespace. The pid will be used as the doc_id, 437 | # which has these requirements. 438 | if not cls.isValidDocId(pid): 439 | raise errors.OperationFailedError("Illegal pid %s" % pid) 440 | # construct the document fields from the params 441 | resfields = cls._buildProductFields( 442 | pid=pid, category=category, name=name, 443 | description=description, 444 | category_name=category_name, price=price, **params) 445 | # build and index the document. Use the pid (product id) as the doc id. 446 | # (If we did not do this, and left the doc_id unspecified, an id would be 447 | # auto-generated.) 448 | d = search.Document(doc_id=pid, fields=resfields) 449 | return d 450 | else: 451 | raise errors.OperationFailedError('Missing parameter.') 452 | 453 | @classmethod 454 | def _normalizeParams(cls, params): 455 | """Normalize the submitted params for building a product.""" 456 | 457 | params = copy.deepcopy(params) 458 | try: 459 | params['pid'] = params['pid'].strip() 460 | params['name'] = params['name'].strip() 461 | params['category_name'] = params['category'] 462 | params['category'] = params['category'] 463 | try: 464 | params['price'] = float(params['price']) 465 | except ValueError: 466 | error_message = 'bad price value: %s' % params['price'] 467 | logging.error(error_message) 468 | raise errors.OperationFailedError(error_message) 469 | return params 470 | except KeyError as e1: 471 | logging.exception("key error") 472 | raise errors.OperationFailedError(e1) 473 | except errors.Error as e2: 474 | logging.debug( 475 | 'Problem with params: %s: %s' % (params, e2.error_message)) 476 | raise errors.OperationFailedError(e2.error_message) 477 | 478 | @classmethod 479 | def buildProductBatch(cls, rows): 480 | """Build product documents and their related datastore entities, in batch, 481 | given a list of params dicts. Should be used for new products, as does not 482 | handle updates of existing product entities. This method does not require 483 | that the doc ids be tied to the product ids, and obtains the doc ids from 484 | the results of the document add.""" 485 | 486 | docs = [] 487 | dbps = [] 488 | for row in rows: 489 | try: 490 | params = cls._normalizeParams(row) 491 | doc = cls._createDocument(**params) 492 | docs.append(doc) 493 | # create product entity, sans doc_id 494 | dbp = models.Product( 495 | id=params['pid'], price=params['price'], 496 | category=params['category']) 497 | dbps.append(dbp) 498 | except errors.OperationFailedError: 499 | logging.error('error creating document from data: %s', row) 500 | try: 501 | add_results = cls.add(docs) 502 | except search.Error: 503 | logging.exception('Add failed') 504 | return 505 | if len(add_results) != len(dbps): 506 | # this case should not be reached; if there was an issue, 507 | # search.Error should have been thrown, above. 508 | raise errors.OperationFailedError( 509 | 'Error: wrong number of results returned from indexing operation') 510 | # now set the entities with the doc ids, the list of which are returned in 511 | # the same order as the list of docs given to the indexers 512 | for i, dbp in enumerate(dbps): 513 | dbp.doc_id = add_results[i].id 514 | # persist the entities 515 | ndb.put_multi(dbps) 516 | 517 | @classmethod 518 | def buildProduct(cls, params): 519 | """Create/update a product document and its related datastore entity. The 520 | product id and the field values are taken from the params dict. 521 | """ 522 | params = cls._normalizeParams(params) 523 | # check to see if doc already exists. We do this because we need to retain 524 | # some information from the existing doc. We could skip the fetch if this 525 | # were not the case. 526 | curr_doc = cls.getDocFromPid(params['pid']) 527 | d = cls._createDocument(**params) 528 | if curr_doc: # retain ratings info from existing doc 529 | avg_rating = cls(curr_doc).getAvgRating() 530 | cls(d).setAvgRating(avg_rating) 531 | 532 | # This will reindex if a doc with that doc id already exists 533 | doc_ids = cls.add(d) 534 | try: 535 | doc_id = doc_ids[0].id 536 | except IndexError: 537 | doc_id = None 538 | raise errors.OperationFailedError('could not index document') 539 | logging.debug('got new doc id %s for product: %s', doc_id, params['pid']) 540 | 541 | # now update the entity 542 | def _tx(): 543 | # Check whether the product entity exists. If so, we want to update 544 | # from the params, but preserve its ratings-related info. 545 | prod = models.Product.get_by_id(params['pid']) 546 | if prod: #update 547 | prod.update_core(params, doc_id) 548 | else: # create new entity 549 | prod = models.Product.create(params, doc_id) 550 | prod.put() 551 | return prod 552 | prod = ndb.transaction(_tx) 553 | logging.debug('prod: %s', prod) 554 | return prod 555 | -------------------------------------------------------------------------------- /product_search_python/errors.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright 2012 Google Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | """Contains the application errors.""" 18 | 19 | 20 | class Error(Exception): 21 | """Base error type.""" 22 | 23 | def __init__(self, error_message): 24 | self.error_message = error_message 25 | 26 | 27 | class NotFoundError(Error): 28 | """Raised when necessary entities are missing.""" 29 | 30 | 31 | class OperationFailedError(Error): 32 | """Raised when necessary operation has failed.""" 33 | 34 | -------------------------------------------------------------------------------- /product_search_python/handlers.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright 2012 Google Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | """Contains the non-admin ('user-facing') request handlers for the app.""" 18 | 19 | 20 | import logging 21 | import urllib 22 | import wsgiref 23 | 24 | from base_handler import BaseHandler 25 | import config 26 | import docs 27 | import models 28 | import utils 29 | 30 | from google.appengine.api import search 31 | from google.appengine.api import users 32 | from google.appengine.ext.deferred import defer 33 | from google.appengine.ext import ndb 34 | 35 | 36 | class IndexHandler(BaseHandler): 37 | """Displays the 'home' page.""" 38 | 39 | def get(self): 40 | cat_info = models.Category.getCategoryInfo() 41 | sort_info = docs.Product.getSortMenu() 42 | template_values = { 43 | 'cat_info': cat_info, 44 | 'sort_info': sort_info, 45 | } 46 | self.render_template('index.html', template_values) 47 | 48 | 49 | class ShowProductHandler(BaseHandler): 50 | """Display product details.""" 51 | 52 | def parseParams(self): 53 | """Filter the param set to the expected params.""" 54 | # The dict can be modified to add any defined defaults. 55 | 56 | params = { 57 | 'pid': '', 58 | 'pname': '', 59 | 'comment': '', 60 | 'rating': '', 61 | 'category': '' 62 | } 63 | for k, v in params.iteritems(): 64 | # Possibly replace default values. 65 | params[k] = self.request.get(k, v) 66 | return params 67 | 68 | def get(self): 69 | """Do a document search for the given product id, 70 | and display the retrieved document fields.""" 71 | 72 | params = self.parseParams() 73 | 74 | pid = params['pid'] 75 | if not pid: 76 | # we should not reach this 77 | msg = 'Error: do not have product id.' 78 | url = '/' 79 | linktext = 'Go to product search page.' 80 | self.render_template( 81 | 'notification.html', 82 | {'title': 'Error', 'msg': msg, 83 | 'goto_url': url, 'linktext': linktext}) 84 | return 85 | doc = docs.Product.getDocFromPid(pid) 86 | if not doc: 87 | error_message = ('Document not found for pid %s.' % pid) 88 | return self.abort(404, error_message) 89 | logging.error(error_message) 90 | pdoc = docs.Product(doc) 91 | pname = pdoc.getName() 92 | app_url = wsgiref.util.application_uri(self.request.environ) 93 | rlink = '/reviews?' + urllib.urlencode({'pid': pid, 'pname': pname}) 94 | template_values = { 95 | 'app_url': app_url, 96 | 'pid': pid, 97 | 'pname': pname, 98 | 'review_link': rlink, 99 | 'comment': params['comment'], 100 | 'rating': params['rating'], 101 | 'category': pdoc.getCategory(), 102 | 'prod_doc': doc, 103 | # for this demo, 'admin' status simply equates to being logged in 104 | 'user_is_admin': users.get_current_user()} 105 | self.render_template('product.html', template_values) 106 | 107 | 108 | class CreateReviewHandler(BaseHandler): 109 | """Process the submission of a new review.""" 110 | 111 | def parseParams(self): 112 | """Filter the param set to the expected params.""" 113 | 114 | params = { 115 | 'pid': '', 116 | 'pname': '', 117 | 'comment': 'this is a great product', 118 | 'rating': '5', 119 | 'category': '' 120 | } 121 | for k, v in params.iteritems(): 122 | # Possibly replace default values. 123 | params[k] = self.request.get(k, v) 124 | return params 125 | 126 | def post(self): 127 | """Create a new review entity from the submitted information.""" 128 | self.createReview(self.parseParams()) 129 | 130 | def createReview(self, params): 131 | """Create a new review entity from the information in the params dict.""" 132 | 133 | author = users.get_current_user() 134 | comment = params['comment'] 135 | pid = params['pid'] 136 | pname = params['pname'] 137 | if not pid: 138 | msg = 'Could not get pid; aborting creation of review.' 139 | logging.error(msg) 140 | url = '/' 141 | linktext = 'Go to product search page.' 142 | self.render_template( 143 | 'notification.html', 144 | {'title': 'Error', 'msg': msg, 145 | 'goto_url': url, 'linktext': linktext}) 146 | return 147 | if not comment: 148 | logging.info('comment not provided') 149 | self.redirect('/product?' + urllib.urlencode(params)) 150 | return 151 | rstring = params['rating'] 152 | # confirm that the rating is an int in the allowed range. 153 | try: 154 | rating = int(rstring) 155 | if rating < config.RATING_MIN or rating > config.RATING_MAX: 156 | logging.warn('Rating %s out of allowed range', rating) 157 | params['rating'] = '' 158 | self.redirect('/product?' + urllib.urlencode(params)) 159 | return 160 | except ValueError: 161 | logging.error('bad rating: %s', rstring) 162 | params['rating'] = '' 163 | self.redirect('/product?' + urllib.urlencode(params)) 164 | return 165 | review = self.createAndAddReview(pid, author, rating, comment) 166 | prod_url = '/product?' + urllib.urlencode({'pid': pid, 'pname': pname}) 167 | if not review: 168 | msg = 'Error creating review.' 169 | logging.error(msg) 170 | self.render_template( 171 | 'notification.html', 172 | {'title': 'Error', 'msg': msg, 173 | 'goto_url': prod_url, 'linktext': 'Back to product.'}) 174 | return 175 | rparams = {'prod_url': prod_url, 'pname': pname, 'review': review} 176 | self.render_template('review.html', rparams) 177 | 178 | def createAndAddReview(self, pid, user, rating, comment): 179 | """Given review information, create the new review entity, pointing via key 180 | to the associated 'parent' product entity. """ 181 | 182 | # get the account info of the user submitting the review. If the 183 | # client is not logged in (which is okay), just make them 'anonymous'. 184 | if user: 185 | username = user.nickname().split('@')[0] 186 | else: 187 | username = 'anonymous' 188 | 189 | prod = models.Product.get_by_id(pid) 190 | if not prod: 191 | error_message = 'could not get product for pid %s' % pid 192 | logging.error(error_message) 193 | return self.abort(404, error_message) 194 | 195 | rid = models.Review.allocate_ids(size=1)[0] 196 | key = ndb.Key(models.Review._get_kind(), rid) 197 | 198 | def _tx(): 199 | review = models.Review( 200 | key=key, 201 | product_key=prod.key, 202 | username=username, rating=rating, 203 | comment=comment) 204 | review.put() 205 | # in a transactional task, update the parent product's average 206 | # rating to include this review's rating, and flag the review as 207 | # processed. 208 | defer(utils.updateAverageRating, key, _transactional=True) 209 | return review 210 | return ndb.transaction(_tx) 211 | 212 | 213 | class ProductSearchHandler(BaseHandler): 214 | """The handler for doing a product search.""" 215 | 216 | _DEFAULT_DOC_LIMIT = 3 #default number of search results to display per page. 217 | _OFFSET_LIMIT = 1000 218 | 219 | def parseParams(self): 220 | """Filter the param set to the expected params.""" 221 | params = { 222 | 'qtype': '', 223 | 'query': '', 224 | 'category': '', 225 | 'sort': '', 226 | 'rating': '', 227 | 'offset': '0' 228 | } 229 | for k, v in params.iteritems(): 230 | # Possibly replace default values. 231 | params[k] = self.request.get(k, v) 232 | return params 233 | 234 | def post(self): 235 | params = self.parseParams() 236 | self.redirect('/psearch?' + urllib.urlencode( 237 | dict([k, v.encode('utf-8')] for k, v in params.items()))) 238 | 239 | def _getDocLimit(self): 240 | """if the doc limit is not set in the config file, use the default.""" 241 | doc_limit = self._DEFAULT_DOC_LIMIT 242 | try: 243 | doc_limit = int(config.DOC_LIMIT) 244 | except ValueError: 245 | logging.error('DOC_LIMIT not properly set in config file; using default.') 246 | return doc_limit 247 | 248 | def get(self): 249 | """Handle a product search request.""" 250 | 251 | params = self.parseParams() 252 | self.doProductSearch(params) 253 | 254 | def doProductSearch(self, params): 255 | """Perform a product search and display the results.""" 256 | 257 | # the defined product categories 258 | cat_info = models.Category.getCategoryInfo() 259 | # the product fields that we can sort on from the UI, and their mappings to 260 | # search.SortExpression parameters 261 | sort_info = docs.Product.getSortMenu() 262 | sort_dict = docs.Product.getSortDict() 263 | query = params.get('query', '') 264 | user_query = query 265 | doc_limit = self._getDocLimit() 266 | 267 | categoryq = params.get('category') 268 | if categoryq: 269 | # add specification of the category to the query 270 | # Because the category field is atomic, put the category string 271 | # in quotes for the search. 272 | query += ' %s:"%s"' % (docs.Product.CATEGORY, categoryq) 273 | 274 | sortq = params.get('sort') 275 | try: 276 | offsetval = int(params.get('offset', 0)) 277 | except ValueError: 278 | offsetval = 0 279 | 280 | # Check to see if the query parameters include a ratings filter, and 281 | # add that to the final query string if so. At the same time, generate 282 | # 'ratings bucket' counts and links-- based on the query prior to addition 283 | # of the ratings filter-- for sidebar display. 284 | query, rlinks = self._generateRatingsInfo( 285 | params, query, user_query, sortq, categoryq) 286 | logging.debug('query: %s', query.strip()) 287 | 288 | try: 289 | # build the query and perform the search 290 | search_query = self._buildQuery( 291 | query, sortq, sort_dict, doc_limit, offsetval) 292 | search_results = docs.Product.getIndex().search(search_query) 293 | returned_count = len(search_results.results) 294 | 295 | except search.Error: 296 | logging.exception("Search error:") # log the exception stack trace 297 | msg = 'There was a search error (see logs).' 298 | url = '/' 299 | linktext = 'Go to product search page.' 300 | self.render_template( 301 | 'notification.html', 302 | {'title': 'Error', 'msg': msg, 303 | 'goto_url': url, 'linktext': linktext}) 304 | return 305 | 306 | # cat_name = models.Category.getCategoryName(categoryq) 307 | psearch_response = [] 308 | # For each document returned from the search 309 | for doc in search_results: 310 | # logging.info("doc: %s ", doc) 311 | pdoc = docs.Product(doc) 312 | # use the description field as the default description snippet, since 313 | # snippeting is not supported on the dev app server. 314 | description_snippet = pdoc.getDescription() 315 | price = pdoc.getPrice() 316 | # on the dev app server, the doc.expressions property won't be populated. 317 | for expr in doc.expressions: 318 | if expr.name == docs.Product.DESCRIPTION: 319 | description_snippet = expr.value 320 | # uncomment to use 'adjusted price', which should be 321 | # defined in returned_expressions in _buildQuery() below, as the 322 | # displayed price. 323 | # elif expr.name == 'adjusted_price': 324 | # price = expr.value 325 | 326 | # get field information from the returned doc 327 | pid = pdoc.getPID() 328 | cat = catname = pdoc.getCategory() 329 | pname = pdoc.getName() 330 | avg_rating = pdoc.getAvgRating() 331 | # for this result, generate a result array of selected doc fields, to 332 | # pass to the template renderer 333 | psearch_response.append( 334 | [doc, urllib.quote_plus(pid), cat, 335 | description_snippet, price, pname, catname, avg_rating]) 336 | if not query: 337 | print_query = 'All' 338 | else: 339 | print_query = query 340 | 341 | # Build the next/previous pagination links for the result set. 342 | (prev_link, next_link) = self._generatePaginationLinks( 343 | offsetval, returned_count, 344 | search_results.number_found, params) 345 | 346 | logging.debug('returned_count: %s', returned_count) 347 | # construct the template values 348 | template_values = { 349 | 'base_pquery': user_query, 'next_link': next_link, 350 | 'prev_link': prev_link, 'qtype': 'product', 351 | 'query': query, 'print_query': print_query, 352 | 'pcategory': categoryq, 'sort_order': sortq, 'category_name': categoryq, 353 | 'first_res': offsetval + 1, 'last_res': offsetval + returned_count, 354 | 'returned_count': returned_count, 355 | 'number_found': search_results.number_found, 356 | 'search_response': psearch_response, 357 | 'cat_info': cat_info, 'sort_info': sort_info, 358 | 'ratings_links': rlinks} 359 | # render the result page. 360 | self.render_template('index.html', template_values) 361 | 362 | def _buildQuery(self, query, sortq, sort_dict, doc_limit, offsetval): 363 | """Build and return a search query object.""" 364 | 365 | # computed and returned fields examples. Their use is not required 366 | # for the application to function correctly. 367 | computed_expr = search.FieldExpression(name='adjusted_price', 368 | expression='price * 1.08') 369 | returned_fields = [docs.Product.PID, docs.Product.DESCRIPTION, 370 | docs.Product.CATEGORY, docs.Product.AVG_RATING, 371 | docs.Product.PRICE, docs.Product.PRODUCT_NAME] 372 | 373 | if sortq == 'relevance': 374 | # If sorting on 'relevance', use the Match scorer. 375 | sortopts = search.SortOptions(match_scorer=search.MatchScorer()) 376 | search_query = search.Query( 377 | query_string=query.strip(), 378 | options=search.QueryOptions( 379 | limit=doc_limit, 380 | offset=offsetval, 381 | sort_options=sortopts, 382 | snippeted_fields=[docs.Product.DESCRIPTION], 383 | returned_expressions=[computed_expr], 384 | returned_fields=returned_fields 385 | )) 386 | else: 387 | # Otherwise (not sorting on relevance), use the selected field as the 388 | # first dimension of the sort expression, and the average rating as the 389 | # second dimension, unless we're sorting on rating, in which case price 390 | # is the second sort dimension. 391 | # We get the sort direction and default from the 'sort_dict' var. 392 | if sortq == docs.Product.AVG_RATING: 393 | expr_list = [sort_dict.get(sortq), sort_dict.get(docs.Product.PRICE)] 394 | else: 395 | expr_list = [sort_dict.get(sortq), sort_dict.get( 396 | docs.Product.AVG_RATING)] 397 | sortopts = search.SortOptions(expressions=expr_list) 398 | # logging.info("sortopts: %s", sortopts) 399 | search_query = search.Query( 400 | query_string=query.strip(), 401 | options=search.QueryOptions( 402 | limit=doc_limit, 403 | offset=offsetval, 404 | sort_options=sortopts, 405 | snippeted_fields=[docs.Product.DESCRIPTION], 406 | returned_expressions=[computed_expr], 407 | returned_fields=returned_fields 408 | )) 409 | return search_query 410 | 411 | def _generateRatingsInfo(self, params, query, user_query, sort, category): 412 | """Add a ratings filter to the query as necessary, and build the 413 | sidebar ratings buckets content.""" 414 | 415 | orig_query = query 416 | try: 417 | n = int(params.get('rating', 0)) 418 | # check that rating is not out of range 419 | if n < config.RATING_MIN or n > config.RATING_MAX: 420 | n = None 421 | except ValueError: 422 | n = None 423 | if n: 424 | if n < config.RATING_MAX: 425 | query += ' %s >= %s %s < %s' % (docs.Product.AVG_RATING, n, 426 | docs.Product.AVG_RATING, n+1) 427 | else: # max rating 428 | query += ' %s:%s' % (docs.Product.AVG_RATING, n) 429 | query_info = {'query': user_query.encode('utf-8'), 'sort': sort, 430 | 'category': category} 431 | rlinks = docs.Product.generateRatingsLinks(orig_query, query_info) 432 | return (query, rlinks) 433 | 434 | def _generatePaginationLinks( 435 | self, offsetval, returned_count, number_found, params): 436 | """Generate the next/prev pagination links for the query. Detect when we're 437 | out of results in a given direction and don't generate the link in that 438 | case.""" 439 | 440 | doc_limit = self._getDocLimit() 441 | pcopy = params.copy() 442 | if offsetval - doc_limit >= 0: 443 | pcopy['offset'] = offsetval - doc_limit 444 | prev_link = '/psearch?' + urllib.urlencode(pcopy) 445 | else: 446 | prev_link = None 447 | if ((offsetval + doc_limit <= self._OFFSET_LIMIT) 448 | and (returned_count == doc_limit) 449 | and (offsetval + returned_count < number_found)): 450 | pcopy['offset'] = offsetval + doc_limit 451 | next_link = '/psearch?' + urllib.urlencode(pcopy) 452 | else: 453 | next_link = None 454 | return (prev_link, next_link) 455 | 456 | 457 | class ShowReviewsHandler(BaseHandler): 458 | """Show the reviews for a given product. This information is pulled from the 459 | datastore Review entities.""" 460 | 461 | def get(self): 462 | """Show a list of reviews for the product indicated by the 'pid' request 463 | parameter.""" 464 | 465 | pid = self.request.get('pid') 466 | pname = self.request.get('pname') 467 | if pid: 468 | # find the product entity corresponding to that pid 469 | prod = models.Product.get_by_id(pid) 470 | if prod: 471 | # get the product's average rating, over all its reviews 472 | # get the list of review entities for the product 473 | avg_rating = prod.avg_rating 474 | reviews = prod.reviews() 475 | logging.debug('reviews: %s', reviews) 476 | else: 477 | error_message = 'could not get product for pid %s' % pid 478 | logging.error(error_message) 479 | return self.abort(404, error_message) 480 | rlist = [[r.username, r.rating, str(r.comment)] for r in reviews] 481 | 482 | # build a template dict with the review and product information 483 | prod_url = '/product?' + urllib.urlencode({'pid': pid, 'pname': pname}) 484 | template_values = { 485 | 'rlist': rlist, 486 | 'prod_url': prod_url, 487 | 'pname': pname, 488 | 'avg_rating': avg_rating} 489 | # render the template. 490 | self.render_template('reviews.html', template_values) 491 | 492 | 493 | class StoreLocationHandler(BaseHandler): 494 | """Show the reviews for a given product. This information is pulled from the 495 | datastore Review entities.""" 496 | 497 | def get(self): 498 | """Show a list of reviews for the product indicated by the 'pid' request 499 | parameter.""" 500 | 501 | query = self.request.get('location_query') 502 | lat = self.request.get('latitude') 503 | lon = self.request.get('longitude') 504 | # the location query from the client will have this form: 505 | # distance(store_location, geopoint(37.7899528, -122.3908226)) < 40000 506 | # logging.info('location query: %s, lat %s, lon %s', query, lat, lon) 507 | try: 508 | index = search.Index(config.STORE_INDEX_NAME) 509 | # search using simply the query string: 510 | # results = index.search(query) 511 | # alternately: sort results by distance 512 | loc_expr = 'distance(store_location, geopoint(%s, %s))' % (lat, lon) 513 | sortexpr = search.SortExpression( 514 | expression=loc_expr, 515 | direction=search.SortExpression.ASCENDING, default_value=0) 516 | sortopts = search.SortOptions(expressions=[sortexpr]) 517 | search_query = search.Query( 518 | query_string=query.strip(), 519 | options=search.QueryOptions( 520 | sort_options=sortopts, 521 | )) 522 | results = index.search(search_query) 523 | except search.Error: 524 | logging.exception("There was a search error:") 525 | self.render_json([]) 526 | return 527 | # logging.info("geo search results: %s", results) 528 | response_obj2 = [] 529 | for res in results: 530 | gdoc = docs.Store(res) 531 | geopoint = gdoc.getFieldVal(gdoc.STORE_LOCATION) 532 | resp = {'addr': gdoc.getFieldVal(gdoc.STORE_ADDRESS), 533 | 'storename': gdoc.getFieldVal(gdoc.STORE_NAME), 534 | 'lat': geopoint.latitude, 'lon': geopoint.longitude} 535 | response_obj2.append(resp) 536 | logging.info("resp: %s", response_obj2) 537 | self.render_json(response_obj2) 538 | -------------------------------------------------------------------------------- /product_search_python/main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright 2011 Google Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | """Defines the routing for the app's non-admin handlers. 18 | """ 19 | 20 | 21 | from handlers import * 22 | import webapp2 23 | 24 | application = webapp2.WSGIApplication( 25 | [('/', IndexHandler), 26 | ('/psearch', ProductSearchHandler), 27 | ('/product', ShowProductHandler), 28 | ('/reviews', ShowReviewsHandler), 29 | ('/create_review', CreateReviewHandler), 30 | ('/get_store_locations', StoreLocationHandler) 31 | ], 32 | debug=True) 33 | 34 | 35 | -------------------------------------------------------------------------------- /product_search_python/models.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright 2012 Google Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | """ Contains the Datastore model classes used by the app: Category, Product, 18 | and Review. 19 | Each Product entity will have a corresponding indexed "product" search.Document. 20 | Product entities contain a subset of the fields in their corresponding document. 21 | Product Review entities are not indexed (do not have corresponding Documents). 22 | Reviews include a product id field, pointing to their 'parent' product, but 23 | are not part of the same entity group, thus avoiding contention in 24 | scenarios where a large number of product reviews might be edited/added at once. 25 | """ 26 | 27 | import logging 28 | 29 | import categories 30 | import docs 31 | 32 | from google.appengine.api import memcache 33 | from google.appengine.ext import ndb 34 | 35 | 36 | class Category(ndb.Model): 37 | """The model class for product category information. Supports building a 38 | category tree.""" 39 | 40 | _CATEGORY_INFO = None 41 | _CATEGORY_DICT = None 42 | _RCATEGORY_DICT = None 43 | _ROOT = 'root' # the 'root' category of the category tree 44 | 45 | parent_category = ndb.KeyProperty() 46 | 47 | @property 48 | def category_name(self): 49 | return self.key.id() 50 | 51 | @classmethod 52 | def buildAllCategories(cls): 53 | """ build the category instances from the provided static data, if category 54 | entities do not already exist in the Datastore. (see categories.py).""" 55 | 56 | # Don't build if there are any categories in the datastore already 57 | if cls.query().get(): 58 | return 59 | root_category = categories.ctree 60 | cls.buildCategory(root_category, None) 61 | 62 | @classmethod 63 | def buildCategory(cls, category_data, parent_key): 64 | """build a category and any children from the given data dict.""" 65 | 66 | if not category_data: 67 | return 68 | cname = category_data.get('name') 69 | if not cname: 70 | logging.warn('no category name for %s', category) 71 | return 72 | if parent_key: 73 | cat = cls(id=cname, parent_category=parent_key) 74 | else: 75 | cat = cls(id=cname) 76 | cat.put() 77 | 78 | children = category_data.get('children') 79 | # if there are any children, build them using their parent key 80 | cls.buildChildCategories(children, cat.key) 81 | 82 | @classmethod 83 | def buildChildCategories(cls, children, parent_key): 84 | """Given a list of category data structures and a parent key, build the 85 | child categories, with the given key as their entity group parent.""" 86 | for cat in children: 87 | cls.buildCategory(cat, parent_key) 88 | 89 | @classmethod 90 | def getCategoryInfo(cls): 91 | """Build and cache a list of category id/name correspondences. This info is 92 | used to populate html select menus.""" 93 | if not cls._CATEGORY_INFO: 94 | cls.buildAllCategories() #first build categories from data file 95 | # if required 96 | cats = cls.query().fetch() 97 | cls._CATEGORY_INFO = [(c.key.id(), c.key.id()) for c in cats 98 | if c.key.id() != cls._ROOT] 99 | return cls._CATEGORY_INFO 100 | 101 | class Product(ndb.Model): 102 | """Model for Product data. A Product entity will be built for each product, 103 | and have an associated search.Document. The product entity does not include 104 | all of the fields in its corresponding indexed product document, only 'core' 105 | fields.""" 106 | 107 | doc_id = ndb.StringProperty() # the id of the associated document 108 | price = ndb.FloatProperty() 109 | category = ndb.StringProperty() 110 | # average rating of the product over all its reviews 111 | avg_rating = ndb.FloatProperty(default=0) 112 | # the number of reviews of that product 113 | num_reviews = ndb.IntegerProperty(default=0) 114 | active = ndb.BooleanProperty(default=True) 115 | # indicates whether the associated document needs to be re-indexed due to a 116 | # change in the average review rating. 117 | needs_review_reindex = ndb.BooleanProperty(default=False) 118 | 119 | @property 120 | def pid(self): 121 | return self.key.id() 122 | 123 | def reviews(self): 124 | """Retrieve all the (active) associated reviews for this product, via the 125 | reviews' product_key field.""" 126 | return Review.query( 127 | Review.active == True, 128 | Review.rating_added == True, 129 | Review.product_key == self.key).fetch() 130 | 131 | @classmethod 132 | def updateProdDocsWithNewRating(cls, pkeys): 133 | """Given a list of product entity keys, check each entity to see if it is 134 | marked as needing a document re-index. This flag is set when a new review 135 | is created for that product, and config.BATCH_RATINGS_UPDATE = True. 136 | Generate the modified docs as needed and batch re-index them.""" 137 | 138 | doclist = [] 139 | 140 | def _tx(pid): 141 | prod = cls.get_by_id(pid) 142 | if prod and prod.needs_review_reindex: 143 | 144 | # update the associated document with the new ratings info 145 | # and reindex 146 | modified_doc = docs.Product.updateRatingInDoc( 147 | prod.doc_id, prod.avg_rating) 148 | if modified_doc: 149 | doclist.append(modified_doc) 150 | prod.needs_review_reindex = False 151 | prod.put() 152 | for pkey in pkeys: 153 | ndb.transaction(lambda: _tx(pkey.id())) 154 | # reindex all modified docs in batch 155 | docs.Product.add(doclist) 156 | 157 | @classmethod 158 | def create(cls, params, doc_id): 159 | """Create a new product entity from a subset of the given params dict 160 | values, and the given doc_id.""" 161 | prod = cls( 162 | id=params['pid'], price=params['price'], 163 | category=params['category'], doc_id=doc_id) 164 | prod.put() 165 | return prod 166 | 167 | def update_core(self, params, doc_id): 168 | """Update 'core' values from the given params dict and doc_id.""" 169 | self.populate( 170 | price=params['price'], category=params['category'], 171 | doc_id=doc_id) 172 | 173 | @classmethod 174 | def updateProdDocWithNewRating(cls, pid): 175 | """Given the id of a product entity, see if it is marked as needing 176 | a document re-index. This flag is set when a new review is created for 177 | that product. If it needs a re-index, call the document method.""" 178 | 179 | def _tx(): 180 | prod = cls.get_by_id(pid) 181 | if prod and prod.needs_review_reindex: 182 | prod.needs_review_reindex = False 183 | prod.put() 184 | return (prod.doc_id, prod.avg_rating) 185 | (doc_id, avg_rating) = ndb.transaction(_tx) 186 | # update the associated document with the new ratings info 187 | # and reindex 188 | docs.Product.updateRatingsInfo(doc_id, avg_rating) 189 | 190 | 191 | class Review(ndb.Model): 192 | """Model for Review data. Associated with a product entity via the product 193 | key.""" 194 | 195 | doc_id = ndb.StringProperty() 196 | date_added = ndb.DateTimeProperty(auto_now_add=True) 197 | product_key = ndb.KeyProperty(kind=Product) 198 | username = ndb.StringProperty() 199 | rating = ndb.IntegerProperty() 200 | active = ndb.BooleanProperty(default=True) 201 | comment = ndb.TextProperty() 202 | rating_added = ndb.BooleanProperty(default=False) 203 | 204 | @classmethod 205 | def deleteReviews(cls, pid): 206 | """Deletes the reviews associated with a product id.""" 207 | if not pid: 208 | return 209 | reviews = cls.query( 210 | cls.product_key == ndb.Key(Product, pid)).fetch(keys_only=True) 211 | return ndb.delete_multi(reviews) 212 | -------------------------------------------------------------------------------- /product_search_python/queue.yaml: -------------------------------------------------------------------------------- 1 | queue: 2 | # - name: default 3 | # rate: 500/s 4 | # bucket_size: 100 5 | -------------------------------------------------------------------------------- /product_search_python/sortoptions.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright 2012 Google Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | import logging 19 | 20 | from google.appengine.api import search 21 | 22 | 23 | def get_sort_options(expressions=None, match_scorer=None, limit=1000): 24 | """A function to handle the sort expression API differences in 1.6.4 25 | vs. 1.6.5+. 26 | 27 | An example of usage (NOTE: Do NOT put limit SortExpression or MatchScorer): 28 | 29 | expr_list = [ 30 | search.SortExpression(expression='author', default_value='', 31 | direction=search.SortExpression.DESCENDING)] 32 | sortopts = get_sort_options(expression=expr_list, limit=sort_limit) 33 | 34 | The returned value is used in constructing the query options: 35 | 36 | qoptions=search.QueryOptions(limit=doc_limit, sort_options=sortopts) 37 | 38 | Another example illustrating sorting on an expression based on a 39 | MatchScorer score: 40 | 41 | expr_list = [ 42 | search.SortExpression(expression='_score + 0.001 * rating', 43 | default_value='', 44 | direction=search.SortExpression.DESCENDING)] 45 | sortopts = get_sort_options(expression=expr_list, 46 | match_scorer=search.MatchScorer(), 47 | limit=sort_limit) 48 | 49 | 50 | Args: 51 | expression: a list of search.SortExpression. Do not set limit parameter on 52 | SortExpression 53 | match_scorer: a search.MatchScorer or search.RescoringMatchScorer. Do not 54 | set limit parameter on either scorer 55 | limit: the scoring limit 56 | 57 | Returns: the sort options value, either list of SortOption (1.6.4) or 58 | SortOptions (1.6.5), to set the sort_options field in the QueryOptions object. 59 | """ 60 | try: 61 | # using 1.6.5 or greater 62 | if search.SortOptions: 63 | logging.debug("search.SortOptions is defined.") 64 | return search.SortOptions( 65 | expressions=expressions, match_scorer=match_scorer, limit=limit) 66 | 67 | # SortOptions not available, so using 1.6.4 68 | except AttributeError: 69 | logging.debug("search.SortOptions is not defined.") 70 | expr_list = [] 71 | # copy the sort expressions including the limit info 72 | if expressions: 73 | expr_list=[ 74 | search.SortExpression( 75 | expression=e.expression, direction=e.direction, 76 | default_value=e.default_value, limit=limit) 77 | for e in expressions] 78 | # add the match scorer, if defined, to the expressions list. 79 | if isinstance(match_scorer, search.MatchScorer): 80 | expr_list.append(match_scorer.__class__(limit=limit)) 81 | logging.info("sort expressions: %s", expr_list) 82 | return expr_list 83 | -------------------------------------------------------------------------------- /product_search_python/static/css/byword.css: -------------------------------------------------------------------------------- 1 | /* 2 | * This document has been created with Marked.app . 3 | * Copyright 2011 Brett Terpstra 4 | * --------------------------------------------------------------------------- 5 | * Please leave this notice in place, along with any additional credits below. 6 | * 7 | * Byword.css theme is based on Byword.app 8 | * Authors: @brunodecarvalho, @jpedroso, @rcabaco 9 | * Copyright 2011 Metaclassy, Lda. 10 | */ 11 | 12 | html { 13 | /*font-size: 62.5%; /* base font-size: 10px */*/ 14 | } 15 | 16 | body { 17 | background-color: #f2f2f2; 18 | color: #3c3c3c; 19 | 20 | /* Change font size below */ 21 | /*font-size: 1.7em; */ 22 | line-height: 1.2em; 23 | 24 | /* Change font below */ 25 | 26 | /* Sans-serif fonts */ 27 | font-family: "Helvetica Neue", Helvetica, Arial, sans-serif; 28 | -webkit-font-smoothing: antialiased; 29 | 30 | /* Serif fonts */ 31 | /* 32 | font-family: "Cochin", "Baskerville", "Georgia", serif; 33 | -webkit-font-smoothing: subpixel-antialiased; 34 | */ 35 | 36 | /* Monospaced fonts */ 37 | /* 38 | font-family: "Courier New", Menlo, Monaco, mono; 39 | -webkit-font-smoothing: antialiased; 40 | */ 41 | } 42 | a { 43 | color: #308bd8; 44 | text-decoration:none; 45 | } 46 | a:hover { 47 | text-decoration: underline; 48 | } 49 | /* headings */ 50 | h1, h2 { 51 | line-height:1.2em; 52 | margin-top:32px; 53 | margin-bottom:12px; 54 | } 55 | h1:first-child { 56 | margin-top:0; 57 | } 58 | h3, h4, h5, h6 { 59 | margin-top:12px; 60 | margin-bottom:0; 61 | } 62 | h5, h6 { 63 | font-size:0.9em; 64 | line-height:1.0em; 65 | } 66 | /* end of headings */ 67 | p { 68 | margin:0 0 24px 0; 69 | } 70 | p:last-child { 71 | margin:0; 72 | } 73 | #wrapper hr { 74 | width: 100%; 75 | margin: 3em auto; 76 | border: 0; 77 | color: #eee; 78 | background-color: #ccc; 79 | height: 1px; 80 | -webkit-box-shadow:0px 1px 0px rgba(255, 255, 255, 0.75); 81 | } 82 | /* lists */ 83 | ol { 84 | list-style: outside decimal; 85 | } 86 | ul { 87 | list-style: outside disc; 88 | } 89 | ol, ul { 90 | padding-left:0; 91 | margin-bottom:24px; 92 | } 93 | ol li { 94 | margin-left:28px; 95 | } 96 | ul li { 97 | margin-bottom:8px; 98 | margin-left:16px; 99 | } 100 | ol:last-child, ul:last-child { 101 | margin:0; 102 | } 103 | li > ol, li > ul { 104 | padding-left:12px; 105 | } 106 | dl { 107 | margin-bottom:24px; 108 | } 109 | dl dt { 110 | font-weight:bold; 111 | margin-bottom:8px; 112 | } 113 | dl dd { 114 | margin-left:0; 115 | margin-bottom:12px; 116 | } 117 | dl dd:last-child, dl:last-child { 118 | margin-bottom:0; 119 | } 120 | /* end of lists */ 121 | pre { 122 | white-space: pre-wrap; 123 | width: 96%; 124 | margin-bottom: 24px; 125 | overflow: hidden; 126 | padding: 3px 10px; 127 | -webkit-border-radius: 3px; 128 | background-color: #eee; 129 | border: 1px solid #ddd; 130 | } 131 | code { 132 | white-space: nowrap; 133 | font-size: 1.1em; 134 | padding: 2px; 135 | -webkit-border-radius: 3px; 136 | background-color: #eee; 137 | border: 1px solid #ddd; 138 | } 139 | pre code { 140 | white-space: pre-wrap; 141 | border: none; 142 | padding: 0; 143 | background-color: transparent; 144 | -webkit-border-radius: 0; 145 | } 146 | blockquote { 147 | margin-left: 0; 148 | margin-right: 0; 149 | width: 96%; 150 | padding: 0 10px; 151 | border-left: 3px solid #ddd; 152 | color: #777; 153 | } 154 | table { 155 | margin-left: auto; 156 | margin-right: auto; 157 | margin-bottom: 24px; 158 | border-bottom: 1px solid #ddd; 159 | border-right: 1px solid #ddd; 160 | border-spacing: 0; 161 | } 162 | table th { 163 | padding: 3px 10px; 164 | background-color: #eee; 165 | border-top: 1px solid #ddd; 166 | border-left: 1px solid #ddd; 167 | } 168 | table tr { 169 | } 170 | table td { 171 | padding: 3px 10px; 172 | border-top: 1px solid #ddd; 173 | border-left: 1px solid #ddd; 174 | } 175 | caption { 176 | font-size: 1.2em; 177 | font-weight: bold; 178 | margin-bottom: 5px; 179 | } 180 | figure { 181 | display: block; 182 | text-align: center; 183 | } 184 | #wrapper img { 185 | border: none; 186 | display: block; 187 | margin: 1em auto; 188 | max-width: 100%; 189 | } 190 | figcaption { 191 | font-size: 0.8em; 192 | font-style: italic; 193 | } 194 | mark { 195 | background: #fefec0; 196 | padding:1px 3px; 197 | } 198 | 199 | 200 | /* classes */ 201 | 202 | .markdowncitation { 203 | } 204 | .footnote { 205 | font-size: 0.8em; 206 | vertical-align: super; 207 | } 208 | .footnotes ol { 209 | font-weight: bold; 210 | } 211 | .footnotes ol li p { 212 | font-weight: normal; 213 | } 214 | 215 | /* custom formatting classes */ 216 | 217 | .shadow { 218 | -webkit-box-shadow: 0 2px 4px #999; 219 | } 220 | 221 | .source { 222 | text-align: center; 223 | font-size: 0.8em; 224 | color: #777; 225 | margin: -40px; 226 | } 227 | 228 | @media screen { 229 | .inverted, .inverted #wrapper { 230 | background-color: #1a1a1a !important; 231 | color: #bebebe !important; 232 | 233 | /* SANS-SERIF */ 234 | font-family: "Helvetica Neue", Helvetica, Arial, sans-serif !important; 235 | -webkit-font-smoothing: antialiased !important; 236 | 237 | /* SERIF */ 238 | /* 239 | font-family: "Cochin", "Baskerville", "Georgia", serif !important; 240 | -webkit-font-smoothing: subpixel-antialiased !important; 241 | */ 242 | /* MONO */ 243 | /* 244 | font-family: "Courier", mono !important; 245 | -webkit-font-smoothing: antialiased !important; 246 | */ 247 | } 248 | .inverted a { 249 | color: #308bd8 !important; 250 | } 251 | .inverted hr { 252 | color: #666 !important; 253 | border: 0; 254 | background-color: #666 !important; 255 | -webkit-box-shadow: none !important; 256 | } 257 | .inverted pre { 258 | background-color: #222 !important; 259 | border-color: #3c3c3c !important; 260 | } 261 | .inverted code { 262 | background-color: #222 !important; 263 | border-color: #3c3c3c !important; 264 | } 265 | .inverted blockquote { 266 | border-color: #333 !important; 267 | color: #999 !important; 268 | } 269 | .inverted table { 270 | border-color: #3c3c3c !important; 271 | } 272 | .inverted table th { 273 | background-color: #222 !important; 274 | border-color: #3c3c3c !important; 275 | } 276 | .inverted table td { 277 | border-color: #3c3c3c !important; 278 | } 279 | .inverted mark { 280 | background: #bc990b !important; 281 | color:#000 !important; 282 | } 283 | .inverted .shadow { -webkit-box-shadow: 0 2px 4px #000 !important; } 284 | #wrapper { 285 | background: transparent; 286 | margin: 40px; 287 | } 288 | } 289 | 290 | /* Printing support */ 291 | @media print { 292 | body { 293 | overflow: auto; 294 | } 295 | img, pre, blockquote, table, figure { 296 | page-break-inside: avoid; 297 | } 298 | pre, code { 299 | border: none !important; 300 | } 301 | #wrapper { 302 | background: #fff; 303 | position: relative; 304 | text-indent: 0px; 305 | padding: 10px; 306 | font-size:85%; 307 | } 308 | .footnotes { 309 | page-break-before: always; 310 | } 311 | } 312 | -------------------------------------------------------------------------------- /product_search_python/static/css/main.css: -------------------------------------------------------------------------------- 1 | body { 2 | font-family: Verdana, Helvetica, sans-serif; 3 | color: #bdd; 4 | background: #255; 5 | padding: 0 5em; 6 | margin: 0 7 | } 8 | 9 | h1 { 10 | padding: 2em 1em; 11 | background: #577 12 | } 13 | 14 | h2 { 15 | color: #add; 16 | border-top: 1px dotted #fff; 17 | margin-top: 2em 18 | } 19 | 20 | a { 21 | color: #9ff 22 | } 23 | 24 | p { 25 | margin: 1em 0 26 | } 27 | -------------------------------------------------------------------------------- /product_search_python/static/instrs.html: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 6 | 7 | 8 | 9 | README 10 | 11 | 12 | 13 |
14 |
15 |

Full-Text Search Demo App: Product Search

16 | 17 |

Introduction

18 | 19 |

This Python App Engine application illustrates the use of the Full-Text Search 20 | API in a “Product 21 | Search” domain with two categories of sample products: books and 22 | hd televisions. This README assumes that you are already familiar with how to 23 | configure and deploy an App Engine app. If not, first see the App Engine 24 | documentation 25 | and Getting Started guide.

26 | 27 |

This demo app allows users to search product information using full-text search, 28 | and to add reviews with ratings to the products. Search results can be sorted 29 | according to several criteria. In conjunction with listed search results, a 30 | sidebar allows the user to further filter the search results on product rating. 31 | (A product’s rating is the average of its reviews, so if a product has no 32 | reviews yet, its rating will be 0).

33 | 34 |

A user does not need to be logged in to search the products, or to add reviews. 35 | A user must be logged in as an admin of the app to add or modify product 36 | data. The sidebar admin links are not displayed for non-admin users.

37 | 38 |

Configuration

39 | 40 |

Before you deploy the application, edit app.yaml to specify your own app id and version.

41 | 42 |

In templates/product.html, the Google Maps API is accessed. It does not require an API key, but you are encouraged to use one to monitor your maps usage. In the element, look for:

43 | 44 |
src="https://maps.googleapis.com/maps/api/js?sensor=false"
 45 | 
46 | 47 |

and replace it with something like the following, where replaceWithYourAPIKey is your own API key:

48 | 49 |
  src="https://maps.googleapis.com/maps/api/js?sensor=false&amp;key=replaceWithYourAPIKey"
 50 | 
51 | 52 |

as described here.

53 | 54 |

Information About Running the App Locally

55 | 56 |

Log in as an app admin to add and modify the app’s product data.

57 | 58 |

The app uses XG (cross-group) transactions, which requires the dev_appserver to 59 | be run with the --high_replication flag. E.g., to start up the dev_appserver 60 | from the command line in the project directory (this directory), assuming the 61 | GAE SDK is in your path, do:

62 | 63 |
dev_appserver.py --high_replication .
 64 | 
65 | 66 |

The app is configured to use Python 2.7. On some platforms, it may also be 67 | necessary to have Python 2.7 installed locally when running the dev_appserver. 68 | The app’s unit tests also require Python 2.7.

69 | 70 |

When running the app locally, not all features of the search API are supported. 71 | So, not all search queries may give the same results during local testing as 72 | when run with the deployed app. 73 | Be sure to test on a deployed version of your app as well as locally.

74 | 75 |

Administering the deployed app

76 | 77 |

You will need to be logged in as an administrator of the app to add and modify 78 | product data, though not to search products or add reviews. If you want to 79 | remove this restriction, you can edit the login: admin specification in 80 | app.yaml, and remove the @BaseHandler.admin decorators in 81 | admin_handlers.py.

82 | 83 |

Loading Sample Data

84 | 85 |

When you first start up your app, you will want to add sample data to it. 86 |

87 | 88 |

Sample product data can be added in two ways. First, sample product data in CSV 89 | format can be added in batch via a link on the app’s admin page. Batch indexing 90 | of documents is more efficient than adding the documents one at a time. For consistency, 91 | the batch addition of sample data first removes all 92 | existing index and datastore product data.

93 | 94 |

The second way to add sample data is via the admin’s “Create new product” link 95 | in the sidebar, which lets an admin add sample products (either “books” or 96 | “hd televisions”) one at a time.

97 | 98 |

Updating product documents with a new average rating

99 | 100 |

When a user creates a new review, the average rating for that product is 101 | updated in the datastore. The app may be configured to update the associated 102 | product search.Document at the same time (the default), or do this at a 103 | later time in batch (which is more efficient). See cron.yaml for an example 104 | of how to do this update periodically in batch.

105 | 106 |

Searches

107 | 108 |

Any valid queries can be typed into the search box. This includes simple word 109 | and phrase queries, but you may also submit queries that include references to 110 | specific document fields and use numeric comparators on numeric fields. See the 111 | Search API’s 112 | documentation for 113 | a description of the query syntax.

114 | 115 |

Thus, for explanatory purposes, the “product details” show all actual 116 | field names of the given product document; you can use this information to 117 | construct queries against those fields. In the same spirit, the raw 118 | query string used for the query is displayed with the search results.

119 | 120 |

Only product information is searched; product review text is not included in the 121 | search.

122 | 123 |

Some example searches

124 | 125 |

Below are some example product queries, which assume the sample data has been loaded. 126 | As discussed above, not all of these queries are supported by the dev_appserver.

127 | 128 |

stories price < 10
129 | price > 10 price < 15
130 | publisher:Vintage
131 | Mega TVs
132 | name:tv1
133 | size > 30

134 | 135 |

Geosearch

136 | 137 |

This application includes an example of using the Search API to perform 138 | location-based queries. Sample store location data is defined in stores.py, 139 | and is loaded along with the product data. The product details page for a 140 | product allows a search for stores within a given radius of the user’s current 141 | location. The user’s location is obtained from the browser.

142 |
143 |
144 | 145 | 146 | -------------------------------------------------------------------------------- /product_search_python/static/js/StyledMarker.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @name StyledMarkerMaker 3 | * @version 0.5 4 | * @author Gabriel Schneider 5 | * @copyright (c) 2010 Gabriel Schneider 6 | * @fileoverview This gives you static functions for creating dynamically 7 | * styled markers using Charts API outputs as well as an ability to 8 | * extend with custom types. 9 | */ 10 | 11 | /** 12 | * Licensed under the Apache License, Version 2.0 (the 'License'); 13 | * you may not use this file except in compliance with the License. 14 | * You may obtain a copy of the License at 15 | * 16 | * http://www.apache.org/licenses/LICENSE-2.0 17 | * 18 | * Unless required by applicable law or agreed to in writing, software 19 | * distributed under the License is distributed on an 'AS IS' BASIS, 20 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 21 | * See the License for the specific language governing permissions and 22 | * limitations under the License. 23 | */ 24 | 25 | var StyledIconTypes = {}; 26 | var StyledMarker, StyledIcon; 27 | 28 | (function() { 29 | var bu_ = 'http://chart.apis.google.com/chart?chst='; 30 | var gm_ = google.maps; 31 | var gp_ = gm_.Point; 32 | var ge_ = gm_.event; 33 | var gmi_ = gm_.MarkerImage; 34 | 35 | 36 | /** 37 | * This class is an extended version of google.maps.Marker. It allows 38 | * styles to be applied that change it's appearance. 39 | * @extends google.maps.Marker 40 | * @param {StyledMarkerOptions} StyledMarkerOptions The options for the Marker 41 | */ 42 | StyledMarker = function(styledMarkerOptions) { 43 | var me=this; 44 | var ci = me.styleIcon = styledMarkerOptions.styleIcon; 45 | me.bindTo('icon',ci); 46 | me.bindTo('shadow',ci); 47 | me.bindTo('shape',ci); 48 | me.setOptions(styledMarkerOptions); 49 | }; 50 | StyledMarker.prototype = new gm_.Marker(); 51 | 52 | /** 53 | * This class stores style information that can be applied to StyledMarkers. 54 | * @extends google.maps.MVCObject 55 | * @param {StyledIconType} styledIconType The type of style this icon is. 56 | * @param {StyledIconOptions} styledIconOptions The options for this StyledIcon. 57 | * @param {StyledIcon} styleClass A class to apply extended style information. 58 | */ 59 | StyledIcon = function(styledIconType,styledIconOptions,styleClass) { 60 | var k; 61 | var me=this; 62 | var i_ = 'icon'; 63 | var sw_ = 'shadow'; 64 | var s_ = 'shape'; 65 | var a_ = []; 66 | 67 | function gs_() { 68 | var image_ = document.createElement('img'); 69 | var simage_ = document.createElement('img'); 70 | ge_.addDomListenerOnce(simage_, 'load', function() { 71 | var w = simage_.width, h = simage_.height; 72 | me.set(sw_,new gmi_(styledIconType.getShadowURL(me),null,null,styledIconType.getShadowAnchor(me,w,h))); 73 | simage = null; 74 | }); 75 | ge_.addDomListenerOnce(image_, 'load', function() { 76 | var w = image_.width, h = image_.height; 77 | me.set(i_,new gmi_(styledIconType.getURL(me),null,null,styledIconType.getAnchor(me,w,h))); 78 | me.set(s_,styledIconType.getShape(me,w,h)); 79 | image_ = null; 80 | }); 81 | image_.src = styledIconType.getURL(me); 82 | simage_.src = styledIconType.getShadowURL(me); 83 | } 84 | 85 | /** 86 | * set: 87 | * This function sets a given style property to the given value. 88 | * @param {String} name The name of the property to set. 89 | * @param {Object} value The value to set the property to. 90 | * get: 91 | * This function gets a given style property. 92 | * @param {String} name The name of the property to get. 93 | * @return {Object} 94 | */ 95 | me.as_ = function(v) { 96 | a_.push(v); 97 | for(k in styledIconOptions) { 98 | v.set(k, styledIconOptions[k]); 99 | } 100 | } 101 | 102 | if (styledIconType !== StyledIconTypes.CLASS) { 103 | for (k in styledIconType.defaults) { 104 | me.set(k, styledIconType.defaults[k]); 105 | } 106 | me.setValues(styledIconOptions); 107 | me.set(i_,styledIconType.getURL(me)); 108 | me.set(sw_,styledIconType.getShadowURL(me)); 109 | if (styleClass) styleClass.as_(me); 110 | gs_(); 111 | me.changed = function(k) { 112 | if (k!==i_&&k!==s_&&k!==sw_) { 113 | gs_(); 114 | } 115 | }; 116 | } else { 117 | me.setValues(styledIconOptions); 118 | me.changed = function(v) { 119 | styledIconOptions[v] = me.get(v); 120 | for (k = 0; k < a_.length; k++) { 121 | a_[k].set(v,me.get(v)); 122 | } 123 | }; 124 | if (styleClass) styleClass.as_(me); 125 | } 126 | }; 127 | StyledIcon.prototype = new gm_.MVCObject(); 128 | 129 | /** 130 | * StyledIconType 131 | * This class holds functions for building the information needed to style markers. 132 | * getURL: 133 | * This function builds and returns a URL to use for the Marker icon property. 134 | * @param {StyledIcon} icon The StyledIcon that holds style information 135 | * @return {String} 136 | * getShadowURL: 137 | * This function builds and returns a URL to use for the Marker shadow property. 138 | * @param {StyledIcon} icon The StyledIcon that holds style information 139 | * @return {String{ 140 | * getAnchor: 141 | * This function builds and returns a Point to indicate where the marker is placed. 142 | * @param {StyledIcon} icon The StyledIcon that holds style information 143 | * @param {Number} width The width of the icon image. 144 | * @param {Number} height The height of the icon image. 145 | * @return {google.maps.Point} 146 | * getShadowAnchor: 147 | * This function builds and returns a Point to indicate where the shadow is placed. 148 | * @param {StyledIcon} icon The StyledIcon that holds style information 149 | * @param {Number} width The width of the shadow image. 150 | * @param {Number} height The height of the shadow image. 151 | * @return {google.maps.Point} 152 | * getShape: 153 | * This function builds and returns a MarkerShape to indicate where the Marker is clickable. 154 | * @param {StyledIcon} icon The StyledIcon that holds style information 155 | * @param {Number} width The width of the icon image. 156 | * @param {Number} height The height of the icon image. 157 | * @return {google.maps.MarkerShape} 158 | */ 159 | 160 | StyledIconTypes.CLASS = {}; 161 | 162 | StyledIconTypes.MARKER = { 163 | defaults: { 164 | text:'', 165 | color:'00ff00', 166 | fore:'000000', 167 | starcolor:null 168 | }, 169 | getURL: function(props){ 170 | var _url; 171 | var starcolor_=props.get('starcolor'); 172 | var text_=props.get('text'); 173 | var color_=props.get('color').replace(/#/,''); 174 | var fore_=props.get('fore').replace(/#/,''); 175 | if (starcolor_) { 176 | _url = bu_ + 'd_map_xpin_letter&chld=pin_star|'; 177 | } else { 178 | _url = bu_ + 'd_map_pin_letter&chld='; 179 | } 180 | if (text_) { 181 | text_ = text_.substr(0,2); 182 | } 183 | _url+=text_+'|'; 184 | _url+=color_+'|'; 185 | _url+=fore_; 186 | if (starcolor_) { 187 | _url+='|'+starcolor_.replace(/#/,''); 188 | } 189 | return _url; 190 | }, 191 | getShadowURL: function(props){ 192 | if (props.get('starcolor')) { 193 | return bu_ + 'd_map_xpin_shadow&chld=pin_star'; 194 | } else { 195 | return bu_ + 'd_map_pin_shadow'; 196 | } 197 | }, 198 | getAnchor: function(props,width,height){ 199 | return new gp_(width / 2,height); 200 | }, 201 | getShadowAnchor: function(props,width,height){ 202 | return new gp_(width / 4,height); 203 | }, 204 | getShape: function(props,width,height){ 205 | var _iconmap = {}; 206 | _iconmap.coord = [ 207 | width / 2, height, 208 | (7 / 16) * width, (5 / 8) * height, 209 | (5 / 16) * width, (7 / 16) * height, 210 | (7 / 32) * width, (5 / 16) * height, 211 | (5 / 16) * width, (1 / 8) * height, 212 | (1 / 2) * width, 0, 213 | (11 / 16) * width, (1 / 8) * height, 214 | (25 / 32) * width, (5 / 16) * height, 215 | (11 / 16) * width, (7 / 16) * height, 216 | (9 / 16) * width, (5 / 8) * height 217 | ]; 218 | for (var i = 0; i < _iconmap.coord.length; i++) { 219 | _iconmap.coord[i] = Math.round(_iconmap.coord[i]); 220 | } 221 | _iconmap.type = 'poly'; 222 | return _iconmap; 223 | } 224 | }; 225 | StyledIconTypes.BUBBLE = { 226 | defaults: { 227 | text:'', 228 | color:'00ff00', 229 | fore:'000000' 230 | }, 231 | getURL: function(props){ 232 | var _url = bu_ + 'd_bubble_text_small&chld=bb|'; 233 | _url+=props.get('text')+'|'; 234 | _url+=props.get('color').replace(/#/,'')+'|'; 235 | _url+=props.get('fore').replace(/#/,''); 236 | return _url; 237 | }, 238 | getShadowURL: function(props){ 239 | return bu_ + 'd_bubble_text_small_shadow&chld=bb|' + props.get('text'); 240 | }, 241 | getAnchor: function(props,width,height){ 242 | return new google.maps.Point(0,42); 243 | }, 244 | getShadowAnchor: function(props,width,height){ 245 | return new google.maps.Point(0,44); 246 | }, 247 | getShape: function(props,width,height){ 248 | var _iconmap = {}; 249 | _iconmap.coord = [ 250 | 0,44, 251 | 13,26, 252 | 13,6, 253 | 17,1, 254 | width - 4,1, 255 | width,6, 256 | width,21, 257 | width - 4,26, 258 | 21,26 259 | ]; 260 | _iconmap.type = 'poly'; 261 | return _iconmap; 262 | } 263 | }; 264 | })(); -------------------------------------------------------------------------------- /product_search_python/stores.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright 2012 Google Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # A set of example retail store locations, specified in terms 18 | # of latitude and longitude. 19 | 20 | stores = [('gosford', 'Gosford', '123 Main St.', 21 | [-33.4282627126087, 151.341658830643]), 22 | ('sydney','Sydney', '123 Main St.', [-33.873038, 151.20563]), 23 | ('marrickville', 'Marrickville', '123 Main St.', 24 | [-33.8950341379958, 151.156479120255]), 25 | ('armidale', 'Armidale', '123 Main St.', [-30.51683, 151.648041]), 26 | ('ashfield', 'Ashfield', '123 Main St.', [-33.888424, 151.124329]), 27 | ('bathurst', 'Bathurst', '123 Main St.', [-33.43528, 149.608887]), 28 | ('blacktown', 'Blacktown', '123 Main St.', [-33.771873, 150.908234]), 29 | ('botany', 'Botany Bay', '123 Main St.', [-33.925842, 151.196564]), 30 | ('london', 'London', '123 Main St.', [51.5000,-0.1167]), 31 | ('paris', 'Paris', '123 Main St.', [48.8667,2.3333]), 32 | ('newyork', 'New York', '123 Main St.', [40.7619,-73.9763]), 33 | ('sanfrancisco', 'San Francisco', '123 Main St.', [37.62, -122.38]), 34 | ('tokyo', 'Tokyo', '123 Main St.', [35.6850, 139.7514]), 35 | ('beijing', 'Beijing', '123 Main St.', [39.9289, 116.3883]), 36 | ('newdelhi', 'New Delhi', '123 Main St.', [28.6000, 77.2000]), 37 | ('lawrence', 'Lawrence', '123 Main St.', [39.0393, -95.2087]), 38 | ('baghdad', 'Baghdad', '123 Main St.', [33.3386, 44.3939]), 39 | ('oakland', 'Oakland', '123 Main St.', [37.73, -122.22]), 40 | ('sancarlos', 'San Carlos', '123 Main St.', [37.52, -122.25]), 41 | ('sanjose', 'San Jose', '123 Main St.', [37.37, -121.92]), 42 | ('hayward', 'Hayward', '123 Main St.', [37.65, -122.12]), 43 | ('monterey', 'Monterey', '123 Main St.', [36.58, -121.85]) 44 | ] 45 | 46 | -------------------------------------------------------------------------------- /product_search_python/templates/admin.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | {% block head %} 3 | Administrative Actions 4 | {% endblock %} 5 | 6 | 7 | {% block content %} 8 | 9 |

Administrative Actions

10 | 11 | {% if notification %} 12 |

Notification: {{notification}}

13 | {% endif %} 14 |

15 | 16 | 28 | 29 | {% endblock %} 30 | 31 | -------------------------------------------------------------------------------- /product_search_python/templates/base.html: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | 7 | 8 | {% block head %}{% endblock %} 9 | 10 | 11 | 12 | 13 | 16 | 17 |
18 | 19 | 39 | 40 |
41 | 42 | {% block content %}{% endblock %} 43 | 44 |
45 | 46 |
47 | 48 | 49 | 50 | -------------------------------------------------------------------------------- /product_search_python/templates/create_product.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | {% block head %} 3 | Add a new product 4 | {% endblock %} 5 | 6 | 7 | 8 | 9 | {% block content %} 10 | 11 |

Add a New Product to the Catalog

12 | 13 |

Enter a unique product ID. It should not contain whitespace. (An id has been auto-generated for you, in case you want to use it).

14 | 15 | {% if error_message %} 16 |

Error: {{error_message}}

17 | {% endif %} 18 | 19 |
20 | 21 |
22 | 23 |

Books

24 | 25 |
26 | 27 | 28 |
29 | 30 |
31 | 32 |
33 |
34 | 35 |
36 | 37 |
38 | 39 |
40 |
41 | 42 |
43 | 44 |
45 | 46 |
47 |
48 | 49 |
50 | 51 |
52 | 53 |
54 |
55 | 56 |
57 | 58 |
59 | 60 |
61 |
62 | 63 |
64 | 65 |
66 | 67 |
68 |
69 | 70 |
71 | 72 |
73 | 74 |
75 |
76 | 77 |
78 | 79 |
80 | 81 |
82 |
83 | 84 |
85 | 86 |
87 | 88 |
89 |
90 | 91 |
92 | 93 | 94 |
95 |
96 | 97 |
98 |
99 | 100 |

Televisions

101 | 102 |
103 | 104 | 105 | 106 |
107 | 108 |
109 | 110 |
111 |
112 | 113 |
114 | 115 |
116 | 117 |
118 |
119 | 120 |
121 | 122 |
123 | 124 |
125 |
126 | 127 |
128 | 129 |
130 | 131 |
132 |
133 | 134 |
135 | 136 |
137 | 138 |
139 |
140 | 141 |
142 | 143 |
144 | 145 |
146 |
147 | 148 |
149 | 150 |
151 | 152 |
153 |
154 | 155 |
156 | 157 | 158 |
159 |
160 | 161 |
162 |
163 | 164 | 165 | {% endblock %} 166 | 167 | -------------------------------------------------------------------------------- /product_search_python/templates/index.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | {% block head %} 3 | Product Search Demo App 4 | {% endblock %} 5 | 6 | 7 | {% block sidebar %} 8 | 9 | 10 | {% if ratings_links %} 11 |

Filter on Rating

12 | 13 |
    14 | {% for elt in ratings_links %} 15 |
  • 16 | {{elt.1}} 17 |
  • 18 | {% endfor %} 19 |
20 | {% endif %} 21 | 22 | {% endblock %} 23 | 24 | 25 | {% block content %} 26 | 27 |

Product Search Demo

28 | 29 |

 

30 | 31 |

Click for information about the demo app. Log in to add sample data.

32 | 33 |

 

34 | 35 | 36 |

Product Search

37 | 38 | 39 |
40 | 41 | 42 | 43 | 44 | 45 | 46 | 59 | 64 | 65 | 78 | 79 | 80 | 84 | 85 |
47 | 57 | 58 | 60 | 61 | 62 | 63 | 66 | 67 | Sort by: 76 | 77 |
81 | 82 | 83 |
86 | 87 |
88 | 89 | 90 | 91 | {% if search_response %} 92 |
93 |

Product Search Results

94 | 95 |

96 | {% if prev_link %} 97 | Previous Results 98 | {% else %} 99 | Previous Results 100 | {% endif %} 101 | | 102 | {% if next_link %} 103 | Next Results 104 | {% else %} 105 | Next Results 106 | {% endif %} 107 |

108 | 109 |

 

110 | {% if returned_count > 0 %} 111 |

112 | {{first_res}} - {{last_res}} of {{number_found}} {{qtype}}s shown for query: {{print_query}}. 113 |

114 | {% endif %} 115 | 116 | {% for result in search_response %} 117 |

118 | Product Description: {{result.3|safe}}
119 | Product name: {{result.5}}
120 | Category: {{result.6}}
121 | Price: {{result.4}}
122 | Average Rating: 123 | {% if result.7 < 1 %} 124 | None yet 125 | {% else %} 126 | {{result.7}} 127 | {% endif %} 128 |
129 | 130 | View product details 131 |  Reviews 132 |

133 | {% endfor %} 134 | 135 |

136 | {% if prev_link %} 137 | Previous Results 138 | {% else %} 139 | Previous Results 140 | {% endif %} 141 | | 142 | {% if next_link %} 143 | Next Results 144 | {% else %} 145 | Next Results 146 | {% endif %} 147 |

148 | 149 |
150 | {%else %} 151 | {%if print_query %} 152 |

No results found.

153 | {% endif %} 154 | 155 | {% endif %} 156 | 157 | 160 | 161 | {% endblock %} 162 | 163 | -------------------------------------------------------------------------------- /product_search_python/templates/notification.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | {% block head %} 3 | {{title}} 4 | {% endblock %} 5 | 6 | {% block content %} 7 | 8 |

9 |

10 | {{msg}} 11 |

12 | {% if url %} 13 |

{{linktext}}

14 | {% endif %} 15 | 16 | {% endblock %} 17 | -------------------------------------------------------------------------------- /product_search_python/templates/product.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | {% block head %} 3 | Product Information for {{pname}} 4 | 5 | 6 | 9 | 10 | 11 | 12 | {% endblock %} 13 | 14 | 15 | {% block content %} 16 | 17 | 148 | 149 |

Product Information for {{pname}}

150 | 151 | 152 | 153 | 154 |
155 | 156 | 158 | 159 |
160 |
161 | 162 |
163 |   units: 164 | 168 |
169 |
170 |
171 | 173 |
174 |
175 |
176 |
177 |
178 | 179 | 180 |


181 | {% for field in prod_doc.fields %} 182 | {{field.name}}: {{field.value}}
183 | {% endfor %} 184 |
Reviews for {{pname}} 185 |

186 | 187 | 188 |

Create a Review for {{pname}}

189 | 190 |
191 | 192 | 193 | 194 |
195 | 196 |
197 | 198 |
199 |
200 | 201 |
202 | 203 |
204 | 205 |
206 |
207 | 208 |
209 | 210 | 211 |
212 |
213 | 214 | {% if user_is_admin %} 215 |

Delete this product and its reviews; no undo (admin).

216 |
217 | 218 |
219 | 220 |
221 |
222 | {% endif %} 223 | 224 | {% endblock %} 225 | -------------------------------------------------------------------------------- /product_search_python/templates/review.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | {% block head %} 3 | Reviews for {{pname}} 4 | {% endblock %} 5 | 6 | {% block content %} 7 | 8 |

9 |

Back to product {{pname}}

10 | 11 |

New review for {{pname}}

12 | 13 |

14 | 15 |

16 | Username: {{review.username}}
17 | Rating: {{review.rating}}
18 | Comment: {{review.comment}}
19 |

20 | 21 | {% endblock %} 22 | -------------------------------------------------------------------------------- /product_search_python/templates/reviews.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | {% block head %} 3 | Reviews for {{pname}} 4 | {% endblock %} 5 | 6 | {% block content %} 7 | 8 |

9 |

Back to product {{pname}}

10 | 11 | 12 |

Reviews for {{pname}}

13 | 14 | {% if rlist %} 15 | 16 |

17 | 18 | Average rating {{avg_rating}}

19 | 20 | {% for result in rlist %} 21 |

22 | Username: {{result.0}}
23 | Rating: {{result.1}}
24 | Comment: {{result.2}}
25 |

26 | {% endfor %} 27 | 28 | {% else %} 29 |

None yet

30 | {% endif %} 31 | 32 | {% endblock %} 33 | -------------------------------------------------------------------------------- /product_search_python/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/appengine-search-python-java/fc8c05e05bc57219e423316dd30d55c3eb4df979/product_search_python/tests/__init__.py -------------------------------------------------------------------------------- /product_search_python/tests/run_unittests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2.7 2 | # 3 | # Copyright 2012 Google Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import optparse 18 | import os 19 | import sys 20 | import unittest 21 | import logging 22 | 23 | USAGE = """%prog SDK_PATH TEST_PATH 24 | Run unit tests for App Engine apps. 25 | 26 | SDK_PATH Path to the SDK installation 27 | TEST_PATH Path to package containing test modules""" 28 | 29 | 30 | def main(sdk_path, test_path): 31 | sys.path.insert(0, sdk_path) 32 | import dev_appserver 33 | dev_appserver.fix_sys_path() 34 | project_dir = os.path.dirname(os.path.dirname(__file__)) 35 | suite = unittest.loader.TestLoader().discover(test_path, 36 | top_level_dir=project_dir) 37 | unittest.TextTestRunner(verbosity=2).run(suite) 38 | 39 | 40 | if __name__ == '__main__': 41 | parser = optparse.OptionParser(USAGE) 42 | options, args = parser.parse_args() 43 | if len(args) != 2: 44 | print 'Error: Exactly 2 arguments required.' 45 | parser.print_help() 46 | sys.exit(1) 47 | sdk_path = args[0] 48 | test_path = args[1] 49 | logging.getLogger().setLevel(logging.ERROR) 50 | main(sdk_path, test_path) 51 | -------------------------------------------------------------------------------- /product_search_python/tests/test_data/testdata.csv: -------------------------------------------------------------------------------- 1 | testproduct 0,The adventures of Sherlock Holmes 0,books,2000,Baker Books 0,The adventures of Sherlock Holmes 0,200,Sir Arthur Conan Doyle 0,The adventures of Sherlock Holmes 0 2 | testproduct 1,The adventures of Sherlock Holmes 1,books,2001,Baker Books 1,The adventures of Sherlock Holmes 1,201,Sir Arthur Conan Doyle 1,The adventures of Sherlock Holmes 1 3 | testproduct 2,The adventures of Sherlock Holmes 2,books,2002,Baker Books 2,The adventures of Sherlock Holmes 2,202,Sir Arthur Conan Doyle 2,The adventures of Sherlock Holmes 2 4 | testproduct 3,The adventures of Sherlock Holmes 3,books,2003,Baker Books 3,The adventures of Sherlock Holmes 3,203,Sir Arthur Conan Doyle 3,The adventures of Sherlock Holmes 3 5 | testproduct 4,The adventures of Sherlock Holmes 4,books,2004,Baker Books 4,The adventures of Sherlock Holmes 4,204,Sir Arthur Conan Doyle 4,The adventures of Sherlock Holmes 4 6 | testproduct 5,The adventures of Sherlock Holmes 5,books,2005,Baker Books 5,The adventures of Sherlock Holmes 5,205,Sir Arthur Conan Doyle 5,The adventures of Sherlock Holmes 5 7 | testproduct 6,The adventures of Sherlock Holmes 6,books,2006,Baker Books 6,The adventures of Sherlock Holmes 6,206,Sir Arthur Conan Doyle 6,The adventures of Sherlock Holmes 6 8 | testproduct 7,The adventures of Sherlock Holmes 7,books,2007,Baker Books 7,The adventures of Sherlock Holmes 7,207,Sir Arthur Conan Doyle 7,The adventures of Sherlock Holmes 7 9 | testproduct 8,The adventures of Sherlock Holmes 8,books,2008,Baker Books 8,The adventures of Sherlock Holmes 8,208,Sir Arthur Conan Doyle 8,The adventures of Sherlock Holmes 8 10 | testproduct 9,The adventures of Sherlock Holmes 9,books,2009,Baker Books 9,The adventures of Sherlock Holmes 9,209,Sir Arthur Conan Doyle 9,The adventures of Sherlock Holmes 9 11 | -------------------------------------------------------------------------------- /product_search_python/tests/test_errors.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright 2012 Google Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | """ Contains unit tests for exceptions.""" 18 | 19 | __author__ = 'tmatsuo@google.com (Takashi Matsuo)' 20 | 21 | 22 | import unittest 23 | 24 | import errors 25 | 26 | 27 | class ErrorTestCase(unittest.TestCase): 28 | 29 | def setUp(self): 30 | pass 31 | 32 | def tearDown(self): 33 | pass 34 | 35 | def testException(self): 36 | error_message = 'It is for test.' 37 | try: 38 | raise errors.NotFoundError(error_message) 39 | except errors.Error as e: 40 | self.assertEqual(error_message, e.error_message) 41 | try: 42 | raise errors.OperationFailedError(error_message) 43 | except errors.Error as e: 44 | self.assertEqual(error_message, e.error_message) 45 | 46 | 47 | if __name__ == '__main__': 48 | unittest.main() 49 | -------------------------------------------------------------------------------- /product_search_python/tests/test_search.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright 2012 Google Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | """ Contains unit tests using search API. 18 | """ 19 | 20 | __author__ = 'tmatsuo@google.com (Takashi Matsuo), amyu@google.com (Amy Unruh)' 21 | 22 | import os 23 | import shutil 24 | import tempfile 25 | import unittest 26 | import base64 27 | import pickle 28 | 29 | from google.appengine.api import apiproxy_stub_map 30 | from google.appengine.api import files 31 | from google.appengine.api import queueinfo 32 | from google.appengine.api import search 33 | from google.appengine.api import users 34 | from google.appengine.api.search import simple_search_stub 35 | from google.appengine.api.taskqueue import taskqueue_stub 36 | from google.appengine.ext import db 37 | from google.appengine.ext import deferred 38 | from google.appengine.ext import testbed 39 | from google.appengine.datastore import datastore_stub_util 40 | 41 | import admin_handlers 42 | import config 43 | import docs 44 | import errors 45 | import models 46 | import utils 47 | 48 | PRODUCT_PARAMS = dict( 49 | pid='testproduct', 50 | name='The adventures of Sherlock Holmes', 51 | category='books', 52 | price=2000, 53 | publisher='Baker Books', 54 | title='The adventures of Sherlock Holmes', 55 | pages=200, 56 | author='Sir Arthur Conan Doyle', 57 | description='The adventures of Sherlock Holmes', 58 | isbn='123456') 59 | 60 | def _add_mark(v, i): 61 | if isinstance(v, basestring): 62 | return '%s %s' % (v, i) 63 | else: 64 | return v + i 65 | 66 | def create_test_data(n): 67 | """Create specified number of test data with marks added to its values.""" 68 | ret = [] 69 | for i in xrange(n): 70 | params = dict() 71 | for key in PRODUCT_PARAMS.keys(): 72 | if key == 'category': 73 | # untouched 74 | params[key] = PRODUCT_PARAMS[key] 75 | else: 76 | params[key] = _add_mark(PRODUCT_PARAMS[key], i) 77 | ret.append(params) 78 | return ret 79 | 80 | 81 | class FTSTestCase(unittest.TestCase): 82 | 83 | def setUp(self): 84 | # First, create an instance of the Testbed class. 85 | self.testbed = testbed.Testbed() 86 | # Then activate the testbed, which prepares the service stubs for use. 87 | self.testbed.activate() 88 | # Create a consistency policy that will simulate the High 89 | # Replication consistency model. It's easier to test with 90 | # probability 1. 91 | self.policy = \ 92 | datastore_stub_util.PseudoRandomHRConsistencyPolicy(probability=1) 93 | # Initialize the datastore stub with this policy. 94 | self.testbed.init_datastore_v3_stub(consistency_policy=self.policy) 95 | self.testbed.init_memcache_stub() 96 | self.testbed.init_taskqueue_stub() 97 | 98 | # search stub is not available via testbed, so doing this by 99 | # myself. 100 | apiproxy_stub_map.apiproxy.RegisterStub( 101 | 'search', 102 | simple_search_stub.SearchServiceStub()) 103 | 104 | def tearDown(self): 105 | self.testbed.deactivate() 106 | 107 | def testBuildProduct(self): 108 | models.Category.buildAllCategories() 109 | self.assertRaises(errors.Error, docs.Product.buildProduct, {}) 110 | 111 | product = docs.Product.buildProduct(PRODUCT_PARAMS) 112 | 113 | # make sure that a product entity is stored in Datastore 114 | self.assert_(product is not None) 115 | self.assertEqual(product.price, PRODUCT_PARAMS['price']) 116 | 117 | # make sure the search actually works 118 | sq = search.Query(query_string='Sir Arthur Conan Doyle') 119 | res = docs.Product.getIndex().search(sq) 120 | self.assertEqual(res.number_found, 1) 121 | for doc in res: 122 | self.assertEqual(doc.doc_id, product.doc_id) 123 | 124 | def testUpdateAverageRatingNonBatch1(self): 125 | "Test non-batch mode avg ratings updating." 126 | models.Category.buildAllCategories() 127 | product = docs.Product.buildProduct(PRODUCT_PARAMS) 128 | self.assertEqual(product.avg_rating, 0) 129 | config.BATCH_RATINGS_UPDATE = False 130 | 131 | # Create a review object and invoke updateAverageRating. 132 | review = models.Review(product_key=product.key, 133 | username='bob', 134 | rating=4, 135 | comment='comment' 136 | ) 137 | review.put() 138 | utils.updateAverageRating(review.key) 139 | review = models.Review(product_key=product.key, 140 | username='bob2', 141 | rating=1, 142 | comment='comment' 143 | ) 144 | review.put() 145 | utils.updateAverageRating(review.key) 146 | 147 | product = models.Product.get_by_id(product.pid) 148 | # check that the parent product rating average has been updated based on the 149 | # two reviews 150 | self.assertEqual(product.avg_rating, 2.5) 151 | # with BATCH_RATINGS_UPDATE = False, the product document's average rating 152 | # field ('ar') should be updated to match its associated product 153 | # entity. 154 | 155 | # run the task queue tasks 156 | taskq = self.testbed.get_stub(testbed.TASKQUEUE_SERVICE_NAME) 157 | tasks = taskq.GetTasks("default") 158 | taskq.FlushQueue("default") 159 | while tasks: 160 | for task in tasks: 161 | deferred.run(base64.b64decode(task["body"])) 162 | tasks = taskq.GetTasks("default") 163 | taskq.FlushQueue("default") 164 | 165 | sq = search.Query(query_string='ar:2.5') 166 | res = docs.Product.getIndex().search(sq) 167 | self.assertEqual(res.number_found, 1) 168 | for doc in res: 169 | self.assertEqual(doc.doc_id, product.doc_id) 170 | 171 | def testUpdateAverageRatingNonBatch2(self): 172 | "Check the number of tasks added to the queue when reviews are created." 173 | 174 | models.Category.buildAllCategories() 175 | product = docs.Product.buildProduct(PRODUCT_PARAMS) 176 | config.BATCH_RATINGS_UPDATE = False 177 | 178 | # Create a review object and invoke updateAverageRating. 179 | review = models.Review(product_key=product.key, 180 | username='bob', 181 | rating=4, 182 | comment='comment' 183 | ) 184 | review.put() 185 | utils.updateAverageRating(review.key) 186 | review = models.Review(product_key=product.key, 187 | username='bob2', 188 | rating=1, 189 | comment='comment' 190 | ) 191 | review.put() 192 | utils.updateAverageRating(review.key) 193 | 194 | # Check the number of tasks in the queue 195 | taskq = self.testbed.get_stub(testbed.TASKQUEUE_SERVICE_NAME) 196 | tasks = taskq.GetTasks("default") 197 | taskq.FlushQueue("default") 198 | self.assertEqual(len(tasks), 2) 199 | 200 | def testUpdateAverageRatingBatch(self): 201 | "Test batch mode avg ratings updating." 202 | models.Category.buildAllCategories() 203 | product = docs.Product.buildProduct(PRODUCT_PARAMS) 204 | config.BATCH_RATINGS_UPDATE = True 205 | 206 | # Create a review object and invoke updateAverageRating. 207 | review = models.Review(product_key=product.key, 208 | username='bob', 209 | rating=5, 210 | comment='comment' 211 | ) 212 | review.put() 213 | utils.updateAverageRating(review.key) 214 | 215 | # there should not be any task queue tasks 216 | taskq = self.testbed.get_stub(testbed.TASKQUEUE_SERVICE_NAME) 217 | tasks = taskq.GetTasks("default") 218 | taskq.FlushQueue("default") 219 | self.assertEqual(len(tasks), 0) 220 | 221 | # with BATCH_RATINGS_UPDATE = True, the product document's average rating 222 | # field ('ar') should not yet be updated to match its associated product 223 | # entity. 224 | product = models.Product.get_by_id(product.pid) 225 | sq = search.Query(query_string='ar:5.0') 226 | res = docs.Product.getIndex().search(sq) 227 | self.assertEqual(res.number_found, 0) 228 | 229 | 230 | if __name__ == '__main__': 231 | unittest.main() 232 | -------------------------------------------------------------------------------- /product_search_python/utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright 2012 Google Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | """Contains utility functions.""" 18 | 19 | import logging 20 | 21 | import config 22 | import docs 23 | import models 24 | 25 | from google.appengine.ext.deferred import defer 26 | from google.appengine.ext import ndb 27 | 28 | 29 | def intClamp(v, low, high): 30 | """Clamps a value to the integer range [low, high] (inclusive). 31 | 32 | Args: 33 | v: Number to be clamped. 34 | low: Lower bound. 35 | high: Upper bound. 36 | 37 | Returns: 38 | An integer closest to v in the range [low, high]. 39 | """ 40 | return max(int(low), min(int(v), int(high))) 41 | 42 | def updateAverageRating(review_key): 43 | """Helper function for updating the average rating of a product when new 44 | review(s) are added.""" 45 | 46 | def _tx(): 47 | review = review_key.get() 48 | product = review.product_key.get() 49 | if not review.rating_added: 50 | review.rating_added = True 51 | product.num_reviews += 1 52 | product.avg_rating = (product.avg_rating + 53 | (review.rating - product.avg_rating)/float(product.num_reviews)) 54 | # signal that we need to reindex the doc with the new ratings info. 55 | product.needs_review_reindex = True 56 | ndb.put_multi([product, review]) 57 | # We need to update the ratings associated document at some point as well. 58 | # If the app is configured to have BATCH_RATINGS_UPDATE set to True, don't 59 | # do this re-indexing now. (Instead, all the out-of-date documents can be 60 | # be later handled in batch -- see cron.yaml). If BATCH_RATINGS_UPDATE is 61 | # False, go ahead and reindex now in a transational task. 62 | if not config.BATCH_RATINGS_UPDATE: 63 | defer( 64 | models.Product.updateProdDocWithNewRating, 65 | product.key.id(), _transactional=True) 66 | return (product, review) 67 | 68 | try: 69 | # use an XG transaction in order to update both entities at once 70 | ndb.transaction(_tx, xg=True) 71 | except AttributeError: 72 | # swallow this error and log it; it's not recoverable. 73 | logging.exception('The function updateAverageRating failed. Either review ' 74 | + 'or product entity does not exist.') 75 | 76 | 77 | 78 | -------------------------------------------------------------------------------- /python/app.yaml: -------------------------------------------------------------------------------- 1 | application: search-py-demo 2 | version: 1 3 | runtime: python27 4 | threadsafe: true 5 | api_version: 1 6 | 7 | handlers: 8 | - url: /static 9 | static_dir: static 10 | 11 | - url: .* 12 | script: search_demo.application 13 | 14 | libraries: 15 | - name: jinja2 16 | version: "2.6" 17 | -------------------------------------------------------------------------------- /python/search_demo.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # 3 | # Copyright 2011 Google Inc. All Rights Reserved. 4 | 5 | """A simple guest book app that demonstrates the App Engine search API.""" 6 | 7 | 8 | from cgi import parse_qs 9 | from datetime import datetime 10 | import os 11 | import string 12 | import urllib 13 | from urlparse import urlparse 14 | 15 | import webapp2 16 | from webapp2_extras import jinja2 17 | 18 | from google.appengine.api import search 19 | from google.appengine.api import users 20 | 21 | _INDEX_NAME = 'greeting' 22 | 23 | # _ENCODE_TRANS_TABLE = string.maketrans('-: .@', '_____') 24 | 25 | class BaseHandler(webapp2.RequestHandler): 26 | """The other handlers inherit from this class. Provides some helper methods 27 | for rendering a template.""" 28 | 29 | @webapp2.cached_property 30 | def jinja2(self): 31 | return jinja2.get_jinja2(app=self.app) 32 | 33 | def render_template(self, filename, template_args): 34 | self.response.write(self.jinja2.render_template(filename, **template_args)) 35 | 36 | 37 | class MainPage(BaseHandler): 38 | """Handles search requests for comments.""" 39 | 40 | def get(self): 41 | """Handles a get request with a query.""" 42 | uri = urlparse(self.request.uri) 43 | query = '' 44 | if uri.query: 45 | query = parse_qs(uri.query) 46 | query = query['query'][0] 47 | 48 | # sort results by author descending 49 | expr_list = [search.SortExpression( 50 | expression='author', default_value='', 51 | direction=search.SortExpression.DESCENDING)] 52 | # construct the sort options 53 | sort_opts = search.SortOptions( 54 | expressions=expr_list) 55 | query_options = search.QueryOptions( 56 | limit=3, 57 | sort_options=sort_opts) 58 | query_obj = search.Query(query_string=query, options=query_options) 59 | results = search.Index(name=_INDEX_NAME).search(query=query_obj) 60 | if users.get_current_user(): 61 | url = users.create_logout_url(self.request.uri) 62 | url_linktext = 'Logout' 63 | else: 64 | url = users.create_login_url(self.request.uri) 65 | url_linktext = 'Login' 66 | 67 | template_values = { 68 | 'results': results, 69 | 'number_returned': len(results.results), 70 | 'url': url, 71 | 'url_linktext': url_linktext, 72 | } 73 | self.render_template('index.html', template_values) 74 | 75 | 76 | def CreateDocument(author, content): 77 | """Creates a search.Document from content written by the author.""" 78 | if author: 79 | nickname = author.nickname().split('@')[0] 80 | else: 81 | nickname = 'anonymous' 82 | # Let the search service supply the document id. 83 | return search.Document( 84 | fields=[search.TextField(name='author', value=nickname), 85 | search.TextField(name='comment', value=content), 86 | search.DateField(name='date', value=datetime.now().date())]) 87 | 88 | 89 | class Comment(BaseHandler): 90 | """Handles requests to index comments.""" 91 | 92 | def post(self): 93 | """Handles a post request.""" 94 | author = None 95 | if users.get_current_user(): 96 | author = users.get_current_user() 97 | 98 | content = self.request.get('content') 99 | query = self.request.get('search') 100 | if content: 101 | search.Index(name=_INDEX_NAME).put(CreateDocument(author, content)) 102 | if query: 103 | self.redirect('/?' + urllib.urlencode( 104 | #{'query': query})) 105 | {'query': query.encode('utf-8')})) 106 | else: 107 | self.redirect('/') 108 | 109 | 110 | application = webapp2.WSGIApplication( 111 | [('/', MainPage), 112 | ('/sign', Comment)], 113 | debug=True) 114 | -------------------------------------------------------------------------------- /python/static/css/main.css: -------------------------------------------------------------------------------- 1 | body { 2 | font-family: Verdana, Helvetica, sans-serif; 3 | color: #333; 4 | background-color: #ddd; 5 | } 6 | -------------------------------------------------------------------------------- /python/templates/index.html: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | Search Demonstration App 7 | 8 | 9 | 10 |
11 |
Search Demo
12 |
13 |
14 |
15 |
16 |
17 | 18 | {{number_returned}} of {{results.number_found}} comments found

19 | {% for scored_document in results %} 20 | {% for f in scored_document.fields %} 21 | {{f.value}}   22 | {% endfor %} 23 |

24 | {% endfor %} 25 | 26 | {{ url_linktext }} 27 | 28 | 29 | 30 | --------------------------------------------------------------------------------