├── .gitignore ├── .ply └── config │ ├── compiler.properties │ ├── dependencies.properties │ ├── dependencies.test.properties │ ├── deploy.properties │ ├── exclusions.properties │ ├── package.properties │ ├── project.properties │ ├── repositories.properties │ └── submodules.properties ├── .travis.yml ├── LICENSE.txt ├── NOTICE.txt ├── README.md ├── geo-core ├── .ply │ └── config │ │ ├── compiler.properties │ │ ├── dependencies.properties │ │ ├── dependencies.test.properties │ │ ├── deploy.properties │ │ ├── exclusions.properties │ │ ├── package.properties │ │ └── project.properties └── src │ ├── main │ └── java │ │ └── com │ │ └── dashlabs │ │ └── dash │ │ └── geo │ │ ├── AbstractGeoQueryHelper.java │ │ ├── model │ │ ├── GeohashRange.java │ │ └── filters │ │ │ ├── GeoDataExtractor.java │ │ │ ├── GeoFilter.java │ │ │ ├── GeoFilters.java │ │ │ ├── RadiusGeoFilter.java │ │ │ └── RectangleGeoFilter.java │ │ └── s2 │ │ └── internal │ │ └── S2Manager.java │ └── test │ └── java │ └── com │ └── dashlabs │ └── dash │ └── geo │ └── s2 │ └── internal │ └── S2ManagerTest.java ├── s3-geo ├── .ply │ └── config │ │ ├── dependencies.properties │ │ ├── deploy.properties │ │ ├── exclusions.properties │ │ ├── package.properties │ │ ├── project.properties │ │ └── repositories.properties └── src │ └── main │ └── java │ └── com │ └── dashlabs │ └── dash │ └── geo │ └── s3 │ ├── Geo.java │ ├── GeoQueryHelper.java │ └── model │ ├── GeoProperties.java │ └── filters │ └── GeoFilters.java └── src ├── main └── java │ └── com │ └── amazonaws │ └── geo │ ├── DefaultHashKeyDecorator.java │ ├── Geo.java │ ├── GeoConfig.java │ ├── GeoQueryHelper.java │ ├── HashKeyDecorator.java │ ├── model │ ├── GeoQueryRequest.java │ └── filters │ │ └── GeoFilters.java │ ├── package-info.java │ └── s2 │ └── internal │ └── GeoQueryClient.java └── test └── java └── com └── amazonaws └── geo ├── GeoQueryClientTest.java └── GeoTest.java /.gitignore: -------------------------------------------------------------------------------- 1 | *.iml 2 | *.ipr 3 | *.iws 4 | target/ 5 | *~ 6 | .DS_Store/ 7 | .DS_Store 8 | gen/ 9 | out/ 10 | .ply/config/local.* 11 | s3-geo/.ply/config/local.* -------------------------------------------------------------------------------- /.ply/config/compiler.properties: -------------------------------------------------------------------------------- 1 | java.source=1.8 2 | java.target=1.8 3 | -------------------------------------------------------------------------------- /.ply/config/dependencies.properties: -------------------------------------------------------------------------------- 1 | com.dashlabs.dash:geo-core=0.0.1 2 | com.amazonaws:aws-java-sdk-dynamodb=1.11.248 3 | -------------------------------------------------------------------------------- /.ply/config/dependencies.test.properties: -------------------------------------------------------------------------------- 1 | junit:junit=4.11 2 | org.mockito:mockito-all=1.9.5 -------------------------------------------------------------------------------- /.ply/config/deploy.properties: -------------------------------------------------------------------------------- 1 | git=${local.hangar51.basedir}/hangar51/repo 2 | -------------------------------------------------------------------------------- /.ply/config/exclusions.properties: -------------------------------------------------------------------------------- 1 | commons-codec:commons-codec=1.3 2 | commons-logging:commons-logging=1.1.3 3 | -------------------------------------------------------------------------------- /.ply/config/package.properties: -------------------------------------------------------------------------------- 1 | includeSrc=true 2 | -------------------------------------------------------------------------------- /.ply/config/project.properties: -------------------------------------------------------------------------------- 1 | namespace=com.amazonaws 2 | name=dynamodb-geo 3 | version=1.1.0 4 | -------------------------------------------------------------------------------- /.ply/config/repositories.properties: -------------------------------------------------------------------------------- 1 | https://api.github.com/repos/Dash-Labs/hangar51/contents/repo=ply 2 | -------------------------------------------------------------------------------- /.ply/config/submodules.properties: -------------------------------------------------------------------------------- 1 | geo-core= 2 | s3-geo= 3 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | jdk: 3 | - oraclejdk8 4 | - openjdk8 5 | install: 6 | - mkdir -p $PWD/opt 7 | - pushd $PWD/opt 8 | - curl https://s3.amazonaws.com/ply-buildtool/ply.tar | tar xz 9 | - ply/bin/ply update 10 | - popd 11 | env: 12 | - PLY_HOME=$PWD/opt/ply PATH=$PLY_HOME/bin:$PATH 13 | script: 14 | - echo $repogithubuser=$repogithubpwd > $PLY_HOME/config/repogithub.properties 15 | - echo https://api.github.com/repos/Dash-Labs/hangar51/contents/repo=$repomngrhangar51 16 | > $PLY_HOME/config/repomngr.properties 17 | - ply clean test 18 | notifications: 19 | email: 20 | - commiters@dash.by 21 | slack: 22 | secure: i3uBEAfws8mxQ/eViAmkM5YI+ptmzV8GzFXv3o5whPSKkhLNarE8yo24KawPtmvlMcgwi0gSVjf+TXEOlB3g/lbjW5doB9o73lT89waJ5blMHs+mVQYHJaMk/NwQmkFuJnhDJDK+z7Grc0vsXedkiGoRT5PwDJQvsbmPCTrG5rEOzMSKnyEus3M+r46Ros9CL1MdmLPEC/W26fHKnC3JKwA25nbB6mi8iwkDOIbWZfklsqJPqvQ7LUBN1fyvFLkqmAXnaipDOSludqJL7Wp40DlBA+EewHp96PKhYDaLHiyYzZG9ahhvXwx4X+F1NlC0Kq+tICYG3fi/x0LkJbObUyDEb+9i9ODPvqD528pBaKhxoztlREcbrEQIFqjzLZWGuYNKxRIgbBloUmwUN2MDYH1aA8eaTFznsF5XNvVT9Gf9INGeyDq39jzfuZnfRtqT+ZlDAwSwJEKOUzqW2Q+s+duwAnRzUOwwkh/nKdNSjSmrPgYIl5j252To8G5TjmBfwB+s1X7OvF/vJnTOCykLFp8kaRkSju0Md2HPWNO05TvpOm+0rZFn5pXAF2tQNCcjQ/1+L/wWOgsy4A/zozU3cXKPWi0iHOrn3KFb0d+RSzym8O8n+WXZDDmpyIU0yQHZZneaNIRiDvD7Ufe8/xkEehZ6weBTE4JlotuYOfAN9QA= 23 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | 4 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 5 | 6 | 1. Definitions. 7 | 8 | "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. 9 | 10 | "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. 11 | 12 | "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. 13 | 14 | "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. 15 | 16 | "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. 17 | 18 | "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. 19 | 20 | "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). 21 | 22 | "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. 23 | 24 | "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." 25 | 26 | "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 27 | 28 | 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 29 | 30 | 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 31 | 32 | 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: 33 | 34 | 1. You must give any other recipients of the Work or Derivative Works a copy of this License; and 35 | 2. You must cause any modified files to carry prominent notices stating that You changed the files; and 36 | 3. You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and 37 | 4. If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. 38 | 39 | You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 40 | 41 | 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 42 | 43 | 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 44 | 45 | 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 46 | 47 | 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 48 | 49 | 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. 50 | 51 | END OF TERMS AND CONDITIONS -------------------------------------------------------------------------------- /NOTICE.txt: -------------------------------------------------------------------------------- 1 | Geo Library for Amazon DynamoDB 2 | Copyright 2010-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | 4 | This product includes software developed by 5 | Amazon Technologies, Inc (http://www.amazon.com/). -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | #Geo Library for Amazon DynamoDB 2 | 3 | [![Build Status](https://travis-ci.com/Dash-Labs/dynamodb-geo.svg?branch=master)](https://travis-ci.com/Dash-Labs/dynamodb-geo) 4 | 5 | This library was forked from the [AWS geo library][geo-library-javadoc]. 6 | 7 | Following limitations with the aws geo library were the main reasons that necessitated this fork: 8 | * Usage required a table’s hash and range key to be replaced by geo data. This approach is not feasible as it cannot be used when performing user-centric queries, where the hash and range key have to be domain model specific attributes. 9 | * Developed prior to GSI, hence only used LSI 10 | * No solution for composite queries. For e.g. “Find something within X miles of lat/lng AND category=‘restaurants’; 11 | * The solution executed the queries and returned the final result. It did not provide the client with any control over the query execution. 12 | 13 | ## What methods are available for geo-querying in this library? 14 | * Query for a given lat/long 15 | * Radius query 16 | * Box/Rectangle query 17 | 18 | All of the above queries can be run as composite queries, depending on their geoConfig. 19 | Result of a geo query is a _GeoQueryRequest_ object. A GeoQueryRequest object is a wrapper around a list of dynamo’s QueryRequest objects and a GeoFilter that should be applied to the queries. 20 | 21 | Benefit of this approach - Callers have the option to execute these queries in a way they desire (multi-threaded, map-reduce jobs, etc). 22 | 23 | ## Creating an item with geo data 24 | * Caller passes in their existing _PutItemRequest_ 25 | * The request gets decorated with the “geo-code” related information and is returned to the client 26 | * YOU control how you want to save your data! 27 | * Bulk persistence 28 | * Large Item persistence strategy (limit on item size in dynamo) 29 | 30 | ##Features 31 | * **Box Queries:** Get a list of _GeoQueryRequest_ objects that will return items that fall within a pair of geo points that define a rectangle as projected onto a sphere. 32 | * **Radius Queries:** Get a list of _GeoQueryRequest_ objects that will return all of the items that are within a given radius of a geo point. 33 | * **Composite Queries:** Get a list of _GeoQueryRequest_ objects that will return all of the items that are within a given radius and has a property 'X' 34 | * **Easy Integration:** The library simply _decorates_ the provided _PutItemRequest_ and _QueryRequest_ with geo-data so you get to control the execution of queries. (multi-threaded, map-reduce jobs, etc) 35 | * **Customizable:** Geo column names and related configuration can be set in the _GeoConfig_ object 36 | 37 | ##Getting Started 38 | ###Setup Environment 39 | 1. **Sign up for AWS** - Before you begin, you need an AWS account. Please see the [AWS Account and Credentials][docs-signup] section of the developer guide for information about how to create an AWS account and retrieve your AWS credentials. 40 | 2. **Download Geo Library for Amazon DynamoDB** - To download the code from GitHub, simply clone the repository by typing: `git clone https://github.com/Dash-Labs/dynamodb-geo.git`. 41 | 42 | ##Building From Source 43 | Once you check out the code from GitHub, you can build it using [Ply](https://github.com/blangel/ply.git): `ply clean install` 44 | 45 | ##Limitations 46 | 47 | ###High I/O needs 48 | Geo query methods will return several queries. Depending on your configuration, this could be thousands of queries 49 | 50 | ###Dataset density limitation 51 | The Geohash used in this library is roughly centimeter precision. Therefore, the library is not suitable if your dataset has much higher density. 52 | 53 | ## Choosing geohash length based on km/m 54 | * http://www.metablake.com/foursquare/wsdm2013-final.pdf - shows S2 lib has 31 levels, giving <1cm^2 at level 31 55 | * http://karussell.wordpress.com/2012/05/23/spatial-keys-memory-efficient-geohashes/ - shows the math to compute for your sphere diameter - diameter / 2^number_levels 56 | * http://unterbahn.com/2009/11/metric-dimensions-of-geohash-partitions-at-the-equator/ - describes how this varies depending upon latitude 57 | * http://ravendb.net/docs/2.0/client-api/querying/static-indexes/spatial-search - gives a great breakdown from 1 - 30 58 | * http://www.easysurf.cc/circle.htm#cetol1 - calculate earth circumference at given latitude 59 | 60 | So, given above, here are some common geohash levels (base 0) mapped to distances: 61 | 62 | Level | dyanmodb-geohash length | Distance 63 | --- | --- | --- 64 | 19 | 12-13 | ~38m at equator (or **~15m** at nyc latitude) 65 | 15 | 10 | ~611m at equator (or **~230m** at nyc latitude) 66 | 9 | 6 |~39km at equator (or **~15km** at nyc latitude) 67 | 6 | 4 | ~313km at equator (or **~118km** at nyc latitude) 68 | 69 | ## Geo querying analysis 70 | 71 | #### New strategy - 10 threads, hashkey length = 5, 20m query followed by 50 m. 72 | Lat/long|Time taken(s) in Dynamo| Time taken(s) in Mongo 73 | ---|---|---| 74 | 40.727526, -73.9944511|0.174|0.054 75 | 38.114560, -117.270763|0.258|0.053 76 | 77 | ### HashKey length - 6: 78 | 79 | Radius(miles) | Number of queries fired 80 | ---|---| 81 | 600|30-->23281 82 | 500|9-->19136 83 | 50|53-->155 84 | 40|31-->105 85 | 30|10-->66 86 | 20|1-->57 87 | 10|6-->10 88 | 1m|1-->1 89 | 90 | ### HashKey length - 5: 91 | 92 | Radius(miles) | Number of queries fired | Time taken(s) | Time taken(s) w/10 threads 93 | ---|---|---|---| 94 | 600|30-->2356 95 | 500|9-->1921 96 | 50|53-->63|1.89|0.37 97 | 40|31-->39|1.134|0.32 98 | 30|10-->16|0.67|0.25 99 | 20|1-->7|0.43|0.19 100 | 10|6-->6|0.27|0.13 101 | 1m|1-->1 102 | 103 | ### HashKey length - 4: 104 | 105 | Radius(miles) | Number of queries fired 106 | ---|---| 107 | 600|30-->262 108 | 500|9-->200 109 | 50|53-->55 110 | 40|31-->33 111 | 30|10-->11 112 | 20|1-->2 113 | 10|6-->6 114 | 1m|1-->1 115 | 116 | ### HashKey length - 3: 117 | 118 | Radius(miles) | Number of queries fired 119 | ---|---| 120 | 600|30-->53 121 | 500|9-->26 122 | 50|53-->54 123 | 40|31-->32 124 | 30|10-->11 125 | 20|1-->2 126 | 10|6-->6 127 | 1m|1-->1 128 | 129 | ### HashKey length - 2: 130 | 131 | Radius(miles) | Number of queries fired 132 | ---|---| 133 | 600|30-->32 134 | 500|9-->10 135 | 50|53-->53 136 | 40|31-->31 137 | 30|10-->10 138 | 20|1-->1 139 | 10|6-->6 140 | 1m|1-->1 141 | 142 | ### HashKey length - 1: 143 | 144 | Radius(miles) | Number of queries fired 145 | ---|---| 146 | 500|9-->9 147 | 50|53-->53 148 | 40|31-->31 149 | 30|10-->10 150 | 20|1-->1 151 | 10|6-->6 152 | 1m|1-->1 153 | 154 | Number of tries to find zip codes (base radius of 10 miles) with hashKey length of 5: 155 | * NYC 40.727526, -73.9944511 : 1 156 | * Haskell County(Texas) 33.215714, -99.814544 : 1 157 | * Torrence County (New Mexico) 34.479959, -105.641418 : 2 158 | * Augusta (Arkansas) 35.292151, -91.260314 : 1 159 | * Tonopah (Nevada)** 38.114560, -117.270763 : 4 160 | * Gypsum(CO)** 39.814929, -106.393789 : 1 161 | 162 | ** For Tonopah, all levels generated 26 queries! 163 | ** For Gypsum(CO)**, 10 miles generated 16 queries, 20 miles generated 30, 30 miles generated 45. 164 | 165 | ##Reference 166 | 167 | ###Amazon DynamoDB 168 | * [Amazon DynamoDB][dynamodb] 169 | * [AWS Geo Library for Amazon DynamoDB] [dynamodb-query] 170 | 171 | [dynamodb]: http://aws.amazon.com/dynamodb 172 | [geo-library-javadoc]: http://awslabs.github.io/dynamodb-geo/ 173 | [dynamodb-query]: http://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_Query.html -------------------------------------------------------------------------------- /geo-core/.ply/config/compiler.properties: -------------------------------------------------------------------------------- 1 | java.target=1.8 2 | java.source=1.8 3 | -------------------------------------------------------------------------------- /geo-core/.ply/config/dependencies.properties: -------------------------------------------------------------------------------- 1 | com.amazonaws:aws-java-sdk-dynamodb=1.11.248 2 | com.google.common.geometry:s2-geometry-java=1.0 3 | org.slf4j:jcl-over-slf4j=1.7.16 4 | -------------------------------------------------------------------------------- /geo-core/.ply/config/dependencies.test.properties: -------------------------------------------------------------------------------- 1 | junit:junit=4.11 2 | org.mockito:mockito-all=1.9.5 3 | -------------------------------------------------------------------------------- /geo-core/.ply/config/deploy.properties: -------------------------------------------------------------------------------- 1 | git=/Users/blangel/projects/dash/hangar51/repo 2 | -------------------------------------------------------------------------------- /geo-core/.ply/config/exclusions.properties: -------------------------------------------------------------------------------- 1 | commons-logging:commons-logging=1.1.3 2 | -------------------------------------------------------------------------------- /geo-core/.ply/config/package.properties: -------------------------------------------------------------------------------- 1 | includeSrc=true 2 | -------------------------------------------------------------------------------- /geo-core/.ply/config/project.properties: -------------------------------------------------------------------------------- 1 | namespace=com.dashlabs.dash 2 | name=geo-core 3 | version=0.0.1 4 | -------------------------------------------------------------------------------- /geo-core/src/main/java/com/dashlabs/dash/geo/AbstractGeoQueryHelper.java: -------------------------------------------------------------------------------- 1 | package com.dashlabs.dash.geo; 2 | 3 | import com.dashlabs.dash.geo.model.GeohashRange; 4 | import com.dashlabs.dash.geo.s2.internal.S2Manager; 5 | import com.google.common.geometry.S2CellId; 6 | import com.google.common.geometry.S2CellUnion; 7 | import com.google.common.geometry.S2LatLngRect; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | import java.util.ArrayList; 12 | import java.util.List; 13 | 14 | /** 15 | * User: blangel 16 | * Date: 8/1/17 17 | * Time: 9:06 AM 18 | */ 19 | public abstract class AbstractGeoQueryHelper { 20 | 21 | private static final Logger LOG = LoggerFactory.getLogger(AbstractGeoQueryHelper.class.getSimpleName()); 22 | 23 | protected final S2Manager s2Manager; 24 | 25 | protected AbstractGeoQueryHelper(S2Manager s2Manager) { 26 | this.s2Manager = s2Manager; 27 | } 28 | 29 | /** 30 | * Creates a collection of GeohashRange by processing each cell {@see com.google.common.geometry.S2CellId} 31 | * that is contained inside the given boundingBox 32 | * 33 | * @param boundingBox the boundingBox {@link com.google.common.geometry.S2LatLngRect} of a given query 34 | * @return ranges a list of GeohashRange 35 | */ 36 | protected List getGeoHashRanges(S2LatLngRect boundingBox) { 37 | S2CellUnion cells = s2Manager.findCellIds(boundingBox); 38 | return mergeCells(cells); 39 | } 40 | 41 | /** 42 | * Merge continuous cells in cellUnion and return a list of merged GeohashRanges. 43 | * 44 | * @param cellUnion Container for multiple cells. 45 | * @return A list of merged GeohashRanges. 46 | */ 47 | protected List mergeCells(S2CellUnion cellUnion) { 48 | List cellIds = cellUnion.cellIds(); 49 | if (cellIds.size() > 1000) { 50 | LOG.warn("Created [{}] cell ids", cellIds.size()); 51 | } 52 | List ranges = new ArrayList<>(cellIds.size()); 53 | for (S2CellId c : cellIds) { 54 | GeohashRange range = new GeohashRange(c.rangeMin().id(), c.rangeMax().id()); 55 | boolean wasMerged = false; 56 | for (GeohashRange r : ranges) { 57 | if (r.tryMerge(range)) { 58 | wasMerged = true; 59 | break; 60 | } 61 | } 62 | if (!wasMerged) { 63 | ranges.add(range); 64 | } 65 | } 66 | return ranges; 67 | } 68 | 69 | } 70 | -------------------------------------------------------------------------------- /geo-core/src/main/java/com/dashlabs/dash/geo/model/GeohashRange.java: -------------------------------------------------------------------------------- 1 | package com.dashlabs.dash.geo.model; 2 | 3 | import com.dashlabs.dash.geo.s2.internal.S2Manager; 4 | 5 | import java.util.ArrayList; 6 | import java.util.List; 7 | 8 | /** 9 | * User: blangel 10 | * Date: 7/19/17 11 | * Time: 1:38 PM 12 | * 13 | * Originally in {@code com.amazonaws.geo.model.GeohashRange} in DynamoDB-Geo (https://github.com/awslabs/dynamodb-geo), 14 | * moved to share across dynamodb-geo and s3 equivalent. 15 | */ 16 | public class GeohashRange { 17 | 18 | public static final long MERGE_THRESHOLD = 2; 19 | 20 | private long rangeMin; 21 | 22 | private long rangeMax; 23 | 24 | 25 | public GeohashRange(long range1, long range2) { 26 | this.rangeMin = Math.min(range1, range2); 27 | this.rangeMax = Math.max(range1, range2); 28 | } 29 | 30 | public boolean tryMerge(GeohashRange range) { 31 | if (range.getRangeMin() - this.rangeMax <= MERGE_THRESHOLD 32 | && range.getRangeMin() - this.rangeMax > 0) { 33 | this.rangeMax = range.getRangeMax(); 34 | return true; 35 | } 36 | 37 | if (this.rangeMin - range.getRangeMax() <= MERGE_THRESHOLD 38 | && this.rangeMin - range.getRangeMax() > 0) { 39 | this.rangeMin = range.getRangeMin(); 40 | return true; 41 | } 42 | 43 | return false; 44 | } 45 | 46 | /* 47 | * Try to split the range to multiple ranges based on the hash key. 48 | * 49 | * e.g., for the following range: 50 | * 51 | * min: 123456789 52 | * max: 125678912 53 | * 54 | * when the hash key length is 3, we want to split the range to: 55 | * 56 | * 1 57 | * min: 123456789 58 | * max: 123999999 59 | * 60 | * 2 61 | * min: 124000000 62 | * max: 124999999 63 | * 64 | * 3 65 | * min: 125000000 66 | * max: 125678912 67 | * 68 | * For this range: 69 | * 70 | * min: -125678912 71 | * max: -123456789 72 | * 73 | * we want: 74 | * 75 | * 1 76 | * min: -125678912 77 | * max: -125000000 78 | * 79 | * 2 80 | * min: -124999999 81 | * max: -124000000 82 | * 83 | * 3 84 | * min: -123999999 85 | * max: -123456789 86 | */ 87 | public List trySplit(int hashKeyLength, S2Manager s2Manager) { 88 | List result = new ArrayList(); 89 | 90 | long minHashKey = s2Manager.generateHashKey(rangeMin, hashKeyLength); 91 | long maxHashKey = s2Manager.generateHashKey(rangeMax, hashKeyLength); 92 | 93 | long denominator = (long) Math.pow(10, String.valueOf(rangeMin).length() - String.valueOf(minHashKey).length()); 94 | 95 | if (minHashKey == maxHashKey) { 96 | result.add(this); 97 | } else { 98 | for (long l = minHashKey; l <= maxHashKey; l++) { 99 | if (l > 0) { 100 | result.add(new GeohashRange(l == minHashKey ? rangeMin : l * denominator, 101 | l == maxHashKey ? rangeMax : (l + 1) * denominator - 1)); 102 | } else { 103 | result.add(new GeohashRange(l == minHashKey ? rangeMin : (l - 1) * denominator + 1, 104 | l == maxHashKey ? rangeMax : l * denominator)); 105 | } 106 | } 107 | } 108 | 109 | return result; 110 | } 111 | 112 | public long getRangeMin() { 113 | return rangeMin; 114 | } 115 | 116 | public void setRangeMin(long rangeMin) { 117 | this.rangeMin = rangeMin; 118 | } 119 | 120 | public long getRangeMax() { 121 | return rangeMax; 122 | } 123 | 124 | public void setRangeMax(long rangeMax) { 125 | this.rangeMax = rangeMax; 126 | } 127 | 128 | } 129 | -------------------------------------------------------------------------------- /geo-core/src/main/java/com/dashlabs/dash/geo/model/filters/GeoDataExtractor.java: -------------------------------------------------------------------------------- 1 | package com.dashlabs.dash.geo.model.filters; 2 | 3 | import java.util.Optional; 4 | 5 | /** 6 | * User: blangel 7 | * Date: 7/19/17 8 | * Time: 2:10 PM 9 | */ 10 | public interface GeoDataExtractor { 11 | 12 | Optional extractLatitude(T item); 13 | 14 | Optional extractLongitude(T item); 15 | 16 | } 17 | -------------------------------------------------------------------------------- /geo-core/src/main/java/com/dashlabs/dash/geo/model/filters/GeoFilter.java: -------------------------------------------------------------------------------- 1 | package com.dashlabs.dash.geo.model.filters; 2 | 3 | import java.util.Collection; 4 | import java.util.List; 5 | 6 | /** 7 | * 8 | * Created originally by mpuri on 3/26/14. 9 | * Represents a filter that can be applied to a collection of items and return a subset of those items. 10 | * 11 | * User: blangel 12 | * Date: 7/19/17 13 | * Time: 2:09 PM 14 | * 15 | * Modified original to abstract the data type 16 | */ 17 | public interface GeoFilter { 18 | 19 | /** 20 | * Fields required for Geo querying 21 | */ 22 | static final String LATITUDE_FIELD = "latitude"; 23 | 24 | static final String LONGITUDE_FIELD = "longitude"; 25 | 26 | /** 27 | * Filters out entities from the given list of items 28 | * 29 | * @param items a list of items that need to be filtered 30 | * @return filteredItems a list containing only the remaining items that did not get filtered. 31 | */ 32 | List filter(Collection items); 33 | 34 | } 35 | -------------------------------------------------------------------------------- /geo-core/src/main/java/com/dashlabs/dash/geo/model/filters/GeoFilters.java: -------------------------------------------------------------------------------- 1 | package com.dashlabs.dash.geo.model.filters; 2 | 3 | import com.google.common.geometry.S2LatLng; 4 | import com.google.common.geometry.S2LatLngRect; 5 | 6 | /** 7 | * 8 | * Originally created by mpuri on 3/26/14. 9 | * Factory methods for {@link GeoFilter}. 10 | * 11 | * User: blangel 12 | * Date: 7/19/17 13 | * Time: 2:11 PM 14 | * 15 | * Modified to abstract data type. 16 | */ 17 | public class GeoFilters { 18 | 19 | /** 20 | * Factory method to create a filter used by radius queries. 21 | * 22 | * @param extractor to extract data from an item 23 | * @param centerLatLng the lat/long of the center of the filter's radius 24 | * @param radiusInMeter the radius of the filter in metres 25 | * @return a new instance of the {@link RadiusGeoFilter} 26 | */ 27 | public static GeoFilter newRadiusFilter(GeoDataExtractor extractor, S2LatLng centerLatLng, double radiusInMeter) { 28 | return new RadiusGeoFilter(extractor, centerLatLng, radiusInMeter); 29 | } 30 | 31 | /** 32 | * Factory method to create a filter used by rectangle queries 33 | * 34 | * @param latLngRect the bounding box for the filter 35 | * @return a new instance of the {@link RectangleGeoFilter} 36 | */ 37 | public static GeoFilter newRectangleFilter(GeoDataExtractor extractor, S2LatLngRect latLngRect) { 38 | return new RectangleGeoFilter(extractor, latLngRect); 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /geo-core/src/main/java/com/dashlabs/dash/geo/model/filters/RadiusGeoFilter.java: -------------------------------------------------------------------------------- 1 | package com.dashlabs.dash.geo.model.filters; 2 | 3 | import com.google.common.geometry.S2LatLng; 4 | 5 | import java.util.ArrayList; 6 | import java.util.Collection; 7 | import java.util.List; 8 | import java.util.Optional; 9 | 10 | /** 11 | * 12 | * Originally created by mpuri on 3/26/14 13 | * 14 | * User: blangel 15 | * Date: 7/19/17 16 | * Time: 2:11 PM 17 | * 18 | * Modified to abstract the data type. 19 | */ 20 | public class RadiusGeoFilter implements GeoFilter { 21 | 22 | private final GeoDataExtractor extractor; 23 | 24 | /** 25 | * Represents a center point as a lat/long - used by radius queries 26 | */ 27 | private final S2LatLng centerLatLng; 28 | 29 | /** 30 | * Radius(in metres) 31 | */ 32 | private final double radiusInMeter; 33 | 34 | public RadiusGeoFilter(GeoDataExtractor extractor, S2LatLng centerLatLng, double radiusInMeter) { 35 | if ((extractor == null) || (centerLatLng == null) || (radiusInMeter <= 0)) { 36 | throw new IllegalArgumentException(); 37 | } 38 | this.extractor = extractor; 39 | this.centerLatLng = centerLatLng; 40 | this.radiusInMeter = radiusInMeter; 41 | } 42 | 43 | /** 44 | * Filters out items that are outside the range of the radius of this filter. 45 | * 46 | * @param items items that need to be filtered. 47 | * @return result a collection of items that fall within the radius of this filter. 48 | */ 49 | public List filter(Collection items) { 50 | List result = new ArrayList<>(items.size()); 51 | for (T item : items) { 52 | Optional latitude = extractor.extractLatitude(item); 53 | Optional longitude = extractor.extractLongitude(item); 54 | if (latitude.isPresent() && longitude.isPresent()) { 55 | S2LatLng latLng = S2LatLng.fromDegrees(latitude.get(), longitude.get()); 56 | if (centerLatLng.getEarthDistance(latLng) <= radiusInMeter) { 57 | result.add(item); 58 | } 59 | } 60 | } 61 | return result; 62 | } 63 | 64 | } 65 | -------------------------------------------------------------------------------- /geo-core/src/main/java/com/dashlabs/dash/geo/model/filters/RectangleGeoFilter.java: -------------------------------------------------------------------------------- 1 | package com.dashlabs.dash.geo.model.filters; 2 | 3 | import com.google.common.geometry.S2LatLng; 4 | import com.google.common.geometry.S2LatLngRect; 5 | 6 | import java.util.ArrayList; 7 | import java.util.Collection; 8 | import java.util.List; 9 | import java.util.Optional; 10 | 11 | /** 12 | * User: blangel 13 | * Date: 7/19/17 14 | * Time: 2:22 PM 15 | */ 16 | public class RectangleGeoFilter implements GeoFilter { 17 | 18 | private final GeoDataExtractor extractor; 19 | 20 | /** 21 | * Bounding box for a rectangle query 22 | */ 23 | private final S2LatLngRect latLngRect; 24 | 25 | public RectangleGeoFilter(GeoDataExtractor extractor, S2LatLngRect latLngRect) { 26 | if ((extractor == null) || (latLngRect == null)) { 27 | throw new IllegalArgumentException(); 28 | } 29 | this.extractor = extractor; 30 | this.latLngRect = latLngRect; 31 | } 32 | 33 | /** 34 | * Filters out items that are outside the range of the bounding box of this filter. 35 | * 36 | * @param items items that need to be filtered. 37 | * @return result a collection of items that fall within the bounding box of this filter. 38 | */ 39 | public List filter(Collection items) { 40 | List result = new ArrayList<>(items.size()); 41 | for (T item : items) { 42 | Optional latitude = extractor.extractLatitude(item); 43 | Optional longitude = extractor.extractLongitude(item); 44 | if (latitude.isPresent() && longitude.isPresent()) { 45 | S2LatLng latLng = S2LatLng.fromDegrees(latitude.get(), longitude.get()); 46 | if (latLngRect.contains(latLng)) { 47 | result.add(item); 48 | } 49 | 50 | } 51 | } 52 | return result; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /geo-core/src/main/java/com/dashlabs/dash/geo/s2/internal/S2Manager.java: -------------------------------------------------------------------------------- 1 | package com.dashlabs.dash.geo.s2.internal; 2 | 3 | import com.google.common.geometry.*; 4 | 5 | import java.util.ArrayList; 6 | import java.util.List; 7 | import java.util.concurrent.ConcurrentLinkedQueue; 8 | 9 | /** 10 | * User: blangel 11 | * Date: 7/19/17 12 | * Time: 1:34 PM 13 | * 14 | * Originally from {@code com.amazonaws.geo.s2.internal.S2Manager} in DynamoDB-Geo (https://github.com/awslabs/dynamodb-geo), 15 | * moved to share across dynamodb-geo and s3 equivalent. 16 | */ 17 | public class S2Manager { 18 | 19 | public S2CellUnion findCellIds(S2LatLngRect latLngRect) { 20 | 21 | ConcurrentLinkedQueue queue = new ConcurrentLinkedQueue(); 22 | ArrayList cellIds = new ArrayList(); 23 | 24 | for (S2CellId c = S2CellId.begin(0); !c.equals(S2CellId.end(0)); c = c.next()) { 25 | if (containsGeodataToFind(c, latLngRect)) { 26 | queue.add(c); 27 | } 28 | } 29 | 30 | processQueue(queue, cellIds, latLngRect); 31 | assert queue.size() == 0; 32 | queue = null; 33 | 34 | if (cellIds.size() > 0) { 35 | S2CellUnion cellUnion = new S2CellUnion(); 36 | cellUnion.initFromCellIds(cellIds); // This normalize the cells. 37 | // cellUnion.initRawCellIds(cellIds); // This does not normalize the cells. 38 | cellIds = null; 39 | 40 | return cellUnion; 41 | } 42 | 43 | return null; 44 | } 45 | 46 | private boolean containsGeodataToFind(S2CellId c, S2LatLngRect latLngRect) { 47 | if (latLngRect != null) { 48 | return latLngRect.intersects(new S2Cell(c)); 49 | } 50 | 51 | return false; 52 | } 53 | 54 | private void processQueue(ConcurrentLinkedQueue queue, ArrayList cellIds, 55 | S2LatLngRect latLngRect) { 56 | for (S2CellId c = queue.poll(); c != null; c = queue.poll()) { 57 | 58 | if (!c.isValid()) { 59 | break; 60 | } 61 | 62 | processChildren(c, latLngRect, queue, cellIds); 63 | } 64 | } 65 | 66 | private void processChildren(S2CellId parent, S2LatLngRect latLngRect, 67 | ConcurrentLinkedQueue queue, ArrayList cellIds) { 68 | List children = new ArrayList(4); 69 | 70 | for (S2CellId c = parent.childBegin(); !c.equals(parent.childEnd()); c = c.next()) { 71 | if (containsGeodataToFind(c, latLngRect)) { 72 | children.add(c); 73 | } 74 | } 75 | 76 | /* 77 | * TODO: Need to update the strategy! 78 | * 79 | * Current strategy: 80 | * 1 or 2 cells contain cellIdToFind: Traverse the children of the cell. 81 | * 3 cells contain cellIdToFind: Add 3 cells for result. 82 | * 4 cells contain cellIdToFind: Add the parent for result. 83 | * 84 | * ** All non-leaf cells contain 4 child cells. 85 | */ 86 | if (children.size() == 1 || children.size() == 2) { 87 | for (S2CellId child : children) { 88 | if (child.isLeaf()) { 89 | cellIds.add(child); 90 | } else { 91 | queue.add(child); 92 | } 93 | } 94 | } else if (children.size() == 3) { 95 | cellIds.addAll(children); 96 | } else if (children.size() == 4) { 97 | cellIds.add(parent); 98 | } else { 99 | assert false; // This should not happen. 100 | } 101 | } 102 | 103 | public long generateGeohash(double latitude, double longitude) { 104 | S2LatLng latLng = S2LatLng.fromDegrees(latitude, longitude); 105 | S2Cell cell = new S2Cell(latLng); 106 | S2CellId cellId = cell.id(); 107 | 108 | return cellId.id(); 109 | } 110 | 111 | public long generateHashKey(long geohash, int hashKeyLength) { 112 | if (geohash < 0) { 113 | // Counteract "-" at beginning of geohash. 114 | hashKeyLength++; 115 | } 116 | 117 | String geohashString = String.valueOf(geohash); 118 | long denominator = (long) Math.pow(10, geohashString.length() - hashKeyLength); 119 | if (denominator == 120 | 0) { // can happen if geohashString.length() < geohash. Querying with a lat/lng of 0.0 can create this situation. 121 | return geohash; 122 | } 123 | return geohash / denominator; 124 | } 125 | 126 | /** 127 | * Creates a bounding box for a radius query 128 | * 129 | * @param latitude the latitude of the radius center 130 | * @param longitude the longitude of the radius center 131 | * @param radius the radius 132 | * @return the bounding box 133 | */ 134 | public S2LatLngRect getBoundingBoxForRadiusQuery(double latitude, double longitude, double radius) { 135 | S2LatLng centerLatLng = S2LatLng.fromDegrees(latitude, longitude); 136 | double latReferenceUnit = latitude > 0.0 ? -1.0 : 1.0; 137 | S2LatLng latReferenceLatLng = S2LatLng.fromDegrees(latitude + latReferenceUnit, 138 | longitude); 139 | double lngReferenceUnit = longitude > 0.0 ? -1.0 : 1.0; 140 | S2LatLng lngReferenceLatLng = S2LatLng.fromDegrees(latitude, longitude 141 | + lngReferenceUnit); 142 | 143 | double latForRadius = radius / centerLatLng.getEarthDistance(latReferenceLatLng); 144 | double lngForRadius = radius / centerLatLng.getEarthDistance(lngReferenceLatLng); 145 | 146 | S2LatLng minLatLng = S2LatLng.fromDegrees(latitude - latForRadius, 147 | longitude - lngForRadius); 148 | S2LatLng maxLatLng = S2LatLng.fromDegrees(latitude + latForRadius, 149 | longitude + lngForRadius); 150 | 151 | return new S2LatLngRect(minLatLng, maxLatLng); 152 | } 153 | 154 | /** 155 | * Creates a bounding box for a rectangle query 156 | * 157 | * @param minLatitude the min latitude of the rectangle 158 | * @param minLongitude the min longitude of the rectangle 159 | * @param maxLatitude the max latitude of the rectangle 160 | * @param maxLongitude the max longitude of the rectangle 161 | * @return the bounding box 162 | */ 163 | public S2LatLngRect getBoundingBoxForRectangleQuery(double minLatitude, double minLongitude, double maxLatitude, double maxLongitude) { 164 | S2LatLng minLatLng = S2LatLng.fromDegrees(minLatitude, minLongitude); 165 | S2LatLng maxLatLng = S2LatLng.fromDegrees(maxLatitude, maxLongitude); 166 | return new S2LatLngRect(minLatLng, maxLatLng); 167 | } 168 | 169 | } 170 | -------------------------------------------------------------------------------- /geo-core/src/test/java/com/dashlabs/dash/geo/s2/internal/S2ManagerTest.java: -------------------------------------------------------------------------------- 1 | package com.dashlabs.dash.geo.s2.internal; 2 | 3 | import org.junit.Test; 4 | 5 | import static org.junit.Assert.assertEquals; 6 | 7 | 8 | /** 9 | * Created by mpuri on 6/2/14 10 | */ 11 | public class S2ManagerTest { 12 | 13 | @Test 14 | public void testGenerateHashKey() { 15 | S2Manager s2Manager = new S2Manager(); 16 | assertEquals(123, s2Manager.generateHashKey(12345678, 3)); 17 | assertEquals(-123, s2Manager.generateHashKey(-12345678, 3)); 18 | assertEquals(12345678, s2Manager.generateHashKey(12345678, 10)); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /s3-geo/.ply/config/dependencies.properties: -------------------------------------------------------------------------------- 1 | com.dashlabs.dash:geo-core=0.0.1 2 | com.amazonaws:aws-java-sdk-dynamodb=1.11.248 3 | -------------------------------------------------------------------------------- /s3-geo/.ply/config/deploy.properties: -------------------------------------------------------------------------------- 1 | git=${local.hangar51.basedir}/hangar51/repo 2 | -------------------------------------------------------------------------------- /s3-geo/.ply/config/exclusions.properties: -------------------------------------------------------------------------------- 1 | commons-logging:commons-logging=1.1.3 2 | -------------------------------------------------------------------------------- /s3-geo/.ply/config/package.properties: -------------------------------------------------------------------------------- 1 | includeSrc=true 2 | -------------------------------------------------------------------------------- /s3-geo/.ply/config/project.properties: -------------------------------------------------------------------------------- 1 | namespace=com.dashlabs.dash 2 | version=0.0.1 3 | name=s3-geo 4 | -------------------------------------------------------------------------------- /s3-geo/.ply/config/repositories.properties: -------------------------------------------------------------------------------- 1 | https://api.github.com/repos/Dash-Labs/hangar51/contents/repo=ply 2 | -------------------------------------------------------------------------------- /s3-geo/src/main/java/com/dashlabs/dash/geo/s3/Geo.java: -------------------------------------------------------------------------------- 1 | package com.dashlabs.dash.geo.s3; 2 | 3 | import com.dashlabs.dash.geo.s3.model.GeoProperties; 4 | import com.dashlabs.dash.geo.model.filters.GeoFilter; 5 | import com.dashlabs.dash.geo.s2.internal.S2Manager; 6 | import com.dashlabs.dash.geo.s3.model.filters.GeoFilters; 7 | import com.google.common.geometry.S2LatLng; 8 | import com.google.common.geometry.S2LatLngRect; 9 | 10 | import java.util.Collection; 11 | import java.util.List; 12 | 13 | /** 14 | * User: blangel 15 | * Date: 7/20/17 16 | * Time: 9:03 AM 17 | */ 18 | public class Geo { 19 | 20 | private final S2Manager s2Manager; 21 | 22 | private final GeoQueryHelper helper; 23 | 24 | public Geo() { 25 | this(new S2Manager()); 26 | } 27 | 28 | private Geo(S2Manager s2Manager) { 29 | this(s2Manager, new GeoQueryHelper(s2Manager)); 30 | } 31 | 32 | protected Geo(S2Manager s2Manager, GeoQueryHelper helper) { 33 | this.s2Manager = s2Manager; 34 | this.helper = helper; 35 | } 36 | 37 | public GeoProperties getGeoProperties(int geoHashLength, double latitude, double longitude) { 38 | long geohash = s2Manager.generateGeohash(latitude, longitude); 39 | long geohashKey = s2Manager.generateHashKey(geohash, geoHashLength); 40 | return new GeoProperties(geoHashLength, geohashKey, latitude, longitude); 41 | } 42 | 43 | public List generatePropertiesForRadiusQuery(int geoHashLength, double latitude, double longitude, double radius) { 44 | S2LatLngRect boundingBox = s2Manager.getBoundingBoxForRadiusQuery(latitude, longitude, radius); 45 | return helper.generateGeoProperties(boundingBox, geoHashLength); 46 | } 47 | 48 | public List generatePropertiesForRectangleQuery(int geoHashLength, double minLatitude, double minLongitude, 49 | double maxLatitude, double maxLongitude) { 50 | S2LatLngRect boundingBox = s2Manager.getBoundingBoxForRectangleQuery(minLatitude, minLongitude, maxLatitude, maxLongitude); 51 | return helper.generateGeoProperties(boundingBox, geoHashLength); 52 | } 53 | 54 | public List filterByRadius(Collection properties, double latitude, double longitude, double radius) { 55 | S2LatLng centerLatLng = S2LatLng.fromDegrees(latitude, longitude); 56 | GeoFilter filter = GeoFilters.newRadiusFilter(centerLatLng, radius); 57 | return filter.filter(properties); 58 | } 59 | 60 | public List filterByRectangle(Collection properties, double minLatitude, double minLongitude, 61 | double maxLatitude, double maxLongitude) { 62 | S2LatLngRect boundingBox = s2Manager.getBoundingBoxForRectangleQuery(minLatitude, minLongitude, maxLatitude, maxLongitude); 63 | GeoFilter filter = GeoFilters.newRectangleFilter(boundingBox); 64 | return filter.filter(properties); 65 | } 66 | 67 | } 68 | -------------------------------------------------------------------------------- /s3-geo/src/main/java/com/dashlabs/dash/geo/s3/GeoQueryHelper.java: -------------------------------------------------------------------------------- 1 | package com.dashlabs.dash.geo.s3; 2 | 3 | import com.dashlabs.dash.geo.AbstractGeoQueryHelper; 4 | import com.dashlabs.dash.geo.model.GeohashRange; 5 | import com.dashlabs.dash.geo.s2.internal.S2Manager; 6 | import com.dashlabs.dash.geo.s3.model.GeoProperties; 7 | import com.google.common.base.Optional; 8 | import com.google.common.collect.ImmutableList; 9 | import com.google.common.geometry.S2CellUnion; 10 | import com.google.common.geometry.S2LatLngRect; 11 | 12 | import java.util.ArrayList; 13 | import java.util.HashMap; 14 | import java.util.List; 15 | import java.util.Map; 16 | 17 | /** 18 | * User: blangel 19 | * Date: 8/1/17 20 | * Time: 9:06 AM 21 | */ 22 | public class GeoQueryHelper extends AbstractGeoQueryHelper { 23 | 24 | public GeoQueryHelper(S2Manager s2Manager) { 25 | super(s2Manager); 26 | } 27 | 28 | /** 29 | * For the given QueryRequest query and the boundingBox, this method creates a collection of queries 30 | * that are decorated with geo attributes to enable geo-spatial querying. 31 | * 32 | * @param boundingBox the bounding lat long rectangle of the geo query 33 | * @param hashKeyLength the hash key length for the geo query 34 | * @return an immutable collection of {@linkplain GeoProperties} 35 | */ 36 | public List generateGeoProperties(S2LatLngRect boundingBox, int hashKeyLength) { 37 | List outerRanges = getGeoHashRanges(boundingBox); 38 | List queryRequests = new ArrayList(outerRanges.size()); 39 | //Create multiple queries based on the geo ranges derived from the bounding box 40 | for (GeohashRange outerRange : outerRanges) { 41 | List geohashRanges = outerRange.trySplit(hashKeyLength, s2Manager); 42 | for (GeohashRange range : geohashRanges) { 43 | long geoHashKey = s2Manager.generateHashKey(range.getRangeMin(), hashKeyLength); 44 | queryRequests.add(new GeoProperties(hashKeyLength, geoHashKey, range.getRangeMin(), range.getRangeMax())); 45 | } 46 | } 47 | return ImmutableList.copyOf(queryRequests); 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /s3-geo/src/main/java/com/dashlabs/dash/geo/s3/model/GeoProperties.java: -------------------------------------------------------------------------------- 1 | package com.dashlabs.dash.geo.s3.model; 2 | 3 | /** 4 | * User: blangel 5 | * Date: 8/1/17 6 | * Time: 8:34 AM 7 | */ 8 | public class GeoProperties { 9 | 10 | private final int hashKeyLength; 11 | 12 | private final long geoHashKey; 13 | 14 | private final double latitude; 15 | 16 | private final double longitude; 17 | 18 | public GeoProperties(int hashKeyLength, long geoHashKey, double latitude, double longitude) { 19 | this.hashKeyLength = hashKeyLength; 20 | this.geoHashKey = geoHashKey; 21 | this.latitude = latitude; 22 | this.longitude = longitude; 23 | } 24 | 25 | public int getHashKeyLength() { 26 | return hashKeyLength; 27 | } 28 | 29 | public long getGeoHashKey() { 30 | return geoHashKey; 31 | } 32 | 33 | public double getLatitude() { 34 | return latitude; 35 | } 36 | 37 | public double getLongitude() { 38 | return longitude; 39 | } 40 | 41 | public String getHashKeyLengthAsString() { 42 | return String.valueOf(hashKeyLength); 43 | } 44 | 45 | public String getGeohashKeyAsString() { 46 | return String.valueOf(geoHashKey); 47 | } 48 | 49 | public String getLatitudeAsString() { 50 | return String.valueOf(latitude); 51 | } 52 | 53 | public String getLongitudeAsString() { 54 | return String.valueOf(longitude); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /s3-geo/src/main/java/com/dashlabs/dash/geo/s3/model/filters/GeoFilters.java: -------------------------------------------------------------------------------- 1 | package com.dashlabs.dash.geo.s3.model.filters; 2 | 3 | import com.dashlabs.dash.geo.model.filters.GeoDataExtractor; 4 | import com.dashlabs.dash.geo.model.filters.GeoFilter; 5 | import com.dashlabs.dash.geo.model.filters.RadiusGeoFilter; 6 | import com.dashlabs.dash.geo.model.filters.RectangleGeoFilter; 7 | import com.dashlabs.dash.geo.s3.model.GeoProperties; 8 | import com.google.common.geometry.S2LatLng; 9 | import com.google.common.geometry.S2LatLngRect; 10 | 11 | import java.util.Optional; 12 | 13 | /** 14 | * User: blangel 15 | * Date: 8/1/17 16 | * Time: 8:46 AM 17 | */ 18 | public class GeoFilters { 19 | 20 | private static final GeoDataExtractor EXTRACTOR = new GeoDataExtractor() { 21 | @Override public Optional extractLatitude(GeoProperties item) { 22 | if (item == null) { 23 | return Optional.empty(); 24 | } 25 | return Optional.of(item.getLatitude()); 26 | } 27 | 28 | @Override public Optional extractLongitude(GeoProperties item) { 29 | if (item == null) { 30 | return Optional.empty(); 31 | } 32 | return Optional.of(item.getLongitude()); 33 | } 34 | }; 35 | 36 | /** 37 | * Factory method to create a filter used by radius queries. 38 | * 39 | * @param centerLatLng the lat/long of the center of the filter's radius 40 | * @param radiusInMeter the radius of the filter in metres 41 | * @return a new instance of the {@link RadiusGeoFilter} 42 | */ 43 | public static GeoFilter newRadiusFilter(S2LatLng centerLatLng, double radiusInMeter) { 44 | return com.dashlabs.dash.geo.model.filters.GeoFilters.newRadiusFilter(EXTRACTOR, centerLatLng, radiusInMeter); 45 | } 46 | 47 | /** 48 | * Factory method to create a filter used by rectangle queries 49 | * 50 | * @param latLngRect the bounding box for the filter 51 | * @return a new instance of the {@link RectangleGeoFilter} 52 | */ 53 | public static GeoFilter newRectangleFilter(S2LatLngRect latLngRect) { 54 | return com.dashlabs.dash.geo.model.filters.GeoFilters.newRectangleFilter(EXTRACTOR, latLngRect); 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /src/main/java/com/amazonaws/geo/DefaultHashKeyDecorator.java: -------------------------------------------------------------------------------- 1 | package com.amazonaws.geo; 2 | 3 | /** 4 | * Created by mpuri on 4/14/14 5 | */ 6 | public class DefaultHashKeyDecorator implements HashKeyDecorator { 7 | 8 | @Override public String decorate(String columnValue, long geoHashKey) { 9 | return String.format("%s:%d", columnValue, geoHashKey); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/main/java/com/amazonaws/geo/Geo.java: -------------------------------------------------------------------------------- 1 | package com.amazonaws.geo; 2 | 3 | import com.amazonaws.geo.model.*; 4 | import com.amazonaws.geo.model.filters.GeoFilters; 5 | import com.dashlabs.dash.geo.model.filters.GeoFilter; 6 | import com.dashlabs.dash.geo.s2.internal.S2Manager; 7 | import com.amazonaws.services.dynamodbv2.model.*; 8 | import com.google.common.base.Optional; 9 | import com.google.common.geometry.S2LatLng; 10 | import com.google.common.geometry.S2LatLngRect; 11 | 12 | import java.util.HashMap; 13 | import java.util.List; 14 | import java.util.Map; 15 | 16 | import static com.google.common.base.Preconditions.checkArgument; 17 | 18 | /** 19 | * Created by mpuri on 3/24/14 20 | */ 21 | public class Geo { 22 | 23 | private final S2Manager s2Manager; 24 | 25 | private final GeoQueryHelper geoQueryHelper; 26 | 27 | public Geo() { 28 | this.s2Manager = new S2Manager(); 29 | this.geoQueryHelper = new GeoQueryHelper(s2Manager); 30 | } 31 | 32 | public Geo(S2Manager s2Manager, GeoQueryHelper geoQueryHelper) { 33 | this.s2Manager = s2Manager; 34 | this.geoQueryHelper = geoQueryHelper; 35 | } 36 | 37 | /** 38 | * Decorates the given putItemRequest with attributes required for geo spatial querying. 39 | * 40 | * @param putItemRequest the request that needs to be decorated with geo attributes 41 | * @param latitude the latitude that needs to be attached with the item 42 | * @param longitude the longitude that needs to be attached with the item 43 | * @param configs the collection of configurations to be used for decorating the request with geo attributes 44 | * @return the decorated request 45 | */ 46 | public PutItemRequest putItemRequest(PutItemRequest putItemRequest, double latitude, double longitude, List configs) { 47 | updateAttributeValues(putItemRequest.getItem(), latitude, longitude, configs); 48 | return putItemRequest; 49 | } 50 | 51 | /** 52 | * Decorates the given updateItemRequest with attributes required for geo spatial querying. 53 | * 54 | * @param attributeValueMap the items that needs to be decorated with geo attributes 55 | * @param latitude the latitude that needs to be attached with the item 56 | * @param longitude the longitude that needs to be attached with the item 57 | * @param configs the collection of configurations to be used for decorating the request with geo attributes 58 | */ 59 | public void updateAttributeValues(Map attributeValueMap, double latitude, double longitude, 60 | List configs) { 61 | if (configs == null) { 62 | throw new IllegalArgumentException("Geo configs should not be null"); 63 | } 64 | for (GeoConfig config : configs) { 65 | //Fail-fast if any of the preconditions fail 66 | checkConfigParams(config.getGeoIndexName(), config.getGeoHashKeyColumn(), config.getGeoHashColumn(), 67 | config.getGeoHashKeyLength()); 68 | 69 | long geohash = s2Manager.generateGeohash(latitude, longitude); 70 | long geoHashKey = s2Manager.generateHashKey(geohash, config.getGeoHashKeyLength()); 71 | 72 | //Decorate the request with the geohash 73 | AttributeValue geoHashValue = new AttributeValue().withN(Long.toString(geohash)); 74 | attributeValueMap.put(config.getGeoHashColumn(), geoHashValue); 75 | 76 | AttributeValue geoHashKeyValue; 77 | if (config.getHashKeyDecorator().isPresent() && config.getCompositeHashKeyColumn().isPresent()) { 78 | AttributeValue compositeHashKeyValue = attributeValueMap.get(config.getCompositeHashKeyColumn().get()); 79 | if (compositeHashKeyValue == null) { 80 | continue; 81 | } 82 | String compositeColumnValue = compositeHashKeyValue.getS(); 83 | String hashKey = config.getHashKeyDecorator().get().decorate(compositeColumnValue, geoHashKey); 84 | //Decorate the request with the composite geoHashKey (type String) 85 | geoHashKeyValue = new AttributeValue().withS(String.valueOf(hashKey)); 86 | } else { 87 | //Decorate the request with the geoHashKey (type Number) 88 | geoHashKeyValue = new AttributeValue().withN(String.valueOf(geoHashKey)); 89 | } 90 | attributeValueMap.put(config.getGeoHashKeyColumn(), geoHashKeyValue); 91 | } 92 | } 93 | 94 | /** 95 | * Decorates the given query request with attributes required for geo spatial querying. 96 | * 97 | * @param queryRequest the request that needs to be decorated with geo attributes 98 | * @param latitude the latitude of the item that is being queried 99 | * @param longitude the longitude of the item that is being queried 100 | * @param config the configuration to be used for decorating the request with geo attributes 101 | * @param compositeKeyValue the value of the column that is used in the construction of the composite hash key(geoHashKey + someOtherColumnValue). 102 | * This is needed when constructing queries that need a composite hash key. 103 | * For eg. Fetch an item where lat/long is 23.78787, -70.6767 AND category = 'restaurants' 104 | * @return the decorated request 105 | */ 106 | public QueryRequest getItemQuery(QueryRequest queryRequest, double latitude, double longitude, GeoConfig config, 107 | Optional compositeKeyValue) { 108 | checkConfigParams(config.getGeoIndexName(), config.getGeoHashKeyColumn(), config.getGeoHashColumn(), config.getGeoHashKeyLength()); 109 | 110 | //Generate the geohash and geoHashKey to query by global secondary index 111 | long geohash = s2Manager.generateGeohash(latitude, longitude); 112 | long geoHashKey = s2Manager.generateHashKey(geohash, config.getGeoHashKeyLength()); 113 | queryRequest.withIndexName(config.getGeoIndexName()); 114 | Map keyConditions = new HashMap(); 115 | 116 | //Construct the hashKey condition 117 | Condition geoHashKeyCondition; 118 | if (config.getHashKeyDecorator().isPresent() && compositeKeyValue.isPresent()) { 119 | String hashKey = config.getHashKeyDecorator().get().decorate(compositeKeyValue.get(), geoHashKey); 120 | geoHashKeyCondition = new Condition().withComparisonOperator(ComparisonOperator.EQ) 121 | .withAttributeValueList(new AttributeValue().withS(hashKey)); 122 | } else { 123 | geoHashKeyCondition = new Condition().withComparisonOperator(ComparisonOperator.EQ) 124 | .withAttributeValueList(new AttributeValue().withN(String.valueOf(geoHashKey))); 125 | } 126 | keyConditions.put(config.getGeoHashKeyColumn(), geoHashKeyCondition); 127 | 128 | //Construct the geohash condition 129 | Condition geoHashCondition = new Condition().withComparisonOperator(ComparisonOperator.EQ) 130 | .withAttributeValueList(new AttributeValue().withN(String.valueOf(geohash))); 131 | keyConditions.put(config.getGeoHashColumn(), geoHashCondition); 132 | 133 | queryRequest.setKeyConditions(keyConditions); 134 | return queryRequest; 135 | } 136 | 137 | /** 138 | * Decorates the given query request with attributes required for geo spatial querying. 139 | * 140 | * @param queryRequest the request that needs to be decorated with geo attributes 141 | * @param latitude the latitude of the item that is being queried 142 | * @param longitude the longitude of the item that is being queried 143 | * @param geoIndexName name of the global secondary index for geo spatial querying 144 | * @param geoHashKeyColumn name of the column that stores the item's geoHashKey. This column is used as a hash key of the global secondary index 145 | * @param geoHashColumn name of the column that stores the item's geohash. This column is used as a range key in the global secondary index 146 | * @param geoHashKeyLength the length of the geohashKey. GeoHashKey is a substring of the item's geohash 147 | * @param compositeKeyValue the value of the column that is used in the construction of the composite hash key(geoHashKey + someOtherColumnValue). 148 | * This is needed when constructing queries that need a composite hash key. 149 | * For eg. Fetch an item where lat/long is 23.78787, -70.6767 AND category = 'restaurants' 150 | * @return the decorated request 151 | */ 152 | public QueryRequest getItemQuery(QueryRequest queryRequest, double latitude, double longitude, String geoIndexName, 153 | String geoHashKeyColumn, String geoHashColumn, 154 | int geoHashKeyLength, Optional compositeKeyValue) { 155 | GeoConfig config = new GeoConfig.Builder().geoHashColumn(geoHashColumn).geoHashKeyColumn(geoHashKeyColumn).geoHashKeyLength( 156 | geoHashKeyLength).geoIndexName(geoIndexName).build(); 157 | return getItemQuery(queryRequest, latitude, longitude, config, compositeKeyValue); 158 | } 159 | 160 | /** 161 | * Creates a wrapper that contains a collection of all queries that are generated as a result of the radius query. 162 | * It also contains a filter {@link com.dashlabs.dash.geo.model.filters.GeoFilter} that needs to be applied to the results of the query 163 | * to ensure that everything is in the radius. 164 | * This is needed because queries are fired for every cell that intersects with the radius' rectangle box. 165 | * 166 | * @param queryRequest the request that needs to be decorated with geo attributes 167 | * @param latitude the latitude of the center point for the radius query 168 | * @param longitude the longitude of the center point for the radius query 169 | * @param radius the radius (in metres) 170 | * @param config the configuration to be used for decorating the request with geo attributes 171 | * @param compositeKeyValue the value of the column that is used in the construction of the composite hash key(geoHashKey + someOtherColumnValue). 172 | * This is needed when constructing queries that need a composite hash key. 173 | * For eg. Fetch an item where lat/long is 23.78787, -70.6767 AND category = 'restaurants' 174 | * @return the wrapper containing the generated queries and the geo filter 175 | */ 176 | public GeoQueryRequest radiusQuery(QueryRequest queryRequest, double latitude, double longitude, double radius, GeoConfig config, Optional compositeKeyValue) { 177 | checkArgument(radius >= 0.0d, "radius has to be a positive value: %s", radius); 178 | checkConfigParams(config.getGeoIndexName(), config.getGeoHashKeyColumn(), config.getGeoHashColumn(), config.getGeoHashKeyLength()); 179 | //Center latLong is needed for the radius filter 180 | S2LatLng centerLatLng = S2LatLng.fromDegrees(latitude, longitude); 181 | GeoFilter> filter = GeoFilters.newRadiusFilter(centerLatLng, radius); 182 | //Bounding box is needed to generate queries for each cell that intersects with the bounding box 183 | S2LatLngRect boundingBox = s2Manager.getBoundingBoxForRadiusQuery(latitude, longitude, radius); 184 | List geoQueries = geoQueryHelper.generateGeoQueries(queryRequest, boundingBox, config, compositeKeyValue); 185 | return new GeoQueryRequest(geoQueries, filter); 186 | } 187 | 188 | /** 189 | * Creates a wrapper that contains a collection of all queries that are generated as a result of the radius query. 190 | * It also contains a filter {@link com.dashlabs.dash.geo.model.filters.GeoFilter} that needs to be applied to the results of the query 191 | * to ensure that everything is in the radius. 192 | * This is needed because queries are fired for every cell that intersects with the radius' rectangle box. 193 | * 194 | * @param queryRequest the request that needs to be decorated with geo attributes 195 | * @param latitude the latitude of the center point for the radius query 196 | * @param longitude the longitude of the center point for the radius query 197 | * @param radius the radius (in metres) 198 | * @param geoIndexName name of the global secondary index for geo spatial querying 199 | * @param geoHashKeyColumn name of the column that stores the item's geoHashKey. This column is used as a hash key of the global secondary index 200 | * @param geoHashColumn name of the column that stores the item's geohash. This column is used as a range key in the global secondary index 201 | * @param geoHashKeyLength the length of the geohashKey. GeoHashKey is a substring of the item's geohash 202 | * @param compositeKeyValue the value of the column that is used in the construction of the composite hash key(geoHashKey + someOtherColumnValue). 203 | * This is needed when constructing queries that need a composite hash key. 204 | * For eg. Fetch an item where lat/long is 23.78787, -70.6767 AND category = 'restaurants' 205 | * @return the wrapper containing the generated queries and the geo filter 206 | */ 207 | public GeoQueryRequest radiusQuery(QueryRequest queryRequest, double latitude, double longitude, double radius, String geoIndexName, 208 | String geoHashKeyColumn, String geoHashColumn, 209 | int geoHashKeyLength, Optional compositeKeyValue) { 210 | GeoConfig config = new GeoConfig.Builder().geoHashColumn(geoHashColumn).geoHashKeyColumn(geoHashKeyColumn).geoHashKeyLength( 211 | geoHashKeyLength).geoIndexName(geoIndexName).build(); 212 | return radiusQuery(queryRequest, latitude, longitude, radius, config, compositeKeyValue); 213 | } 214 | 215 | /** 216 | * Creates a wrapper that contains a collection of all queries that are generated as a result of this rectangle query. 217 | * It also contains a filter {@link com.dashlabs.dash.geo.model.filters.GeoFilter} that needs to be applied to the results of the query 218 | * to ensure that everything is in the bounding box of the queried rectangle. 219 | * This is needed because queries are fired for every cell that intersects with the rectangle's bounding box. 220 | * 221 | * @param queryRequest the request that needs to be decorated with geo attributes 222 | * @param minLatitude the latitude of the min point of the rectangle 223 | * @param minLongitude the longitude of the min point of the rectangle 224 | * @param maxLatitude the latitude of the max point of the rectangle 225 | * @param maxLongitude the longitude of the max point of the rectangle 226 | * @param config the configuration to be used for decorating the request with geo attributes 227 | * @param compositeKeyValue the value of the column that is used in the construction of the composite hash key(geoHashKey + someOtherColumnValue). 228 | * This is needed when constructing queries that need a composite hash key. 229 | * For eg. Fetch an item where lat/long is 23.78787, -70.6767 AND category = 'restaurants' 230 | * @return the wrapper containing the generated queries and the geo filter 231 | */ 232 | public GeoQueryRequest rectangleQuery(QueryRequest queryRequest, double minLatitude, double minLongitude, double maxLatitude, 233 | double maxLongitude, GeoConfig config, Optional compositeKeyValue) { 234 | checkConfigParams(config.getGeoIndexName(), config.getGeoHashKeyColumn(), config.getGeoHashColumn(), config.getGeoHashKeyLength()); 235 | // bounding box is needed for the filter and to generate the queries 236 | // for each cell that intersects with the bounding box 237 | S2LatLngRect boundingBox = s2Manager.getBoundingBoxForRectangleQuery(minLatitude, minLongitude, maxLatitude, maxLongitude); 238 | GeoFilter> filter = GeoFilters.newRectangleFilter(boundingBox); 239 | List geoQueries = geoQueryHelper.generateGeoQueries(queryRequest, boundingBox, config, compositeKeyValue); 240 | return new GeoQueryRequest(geoQueries, filter); 241 | } 242 | 243 | /** 244 | * Creates a wrapper that contains a collection of all queries that are generated as a result of this rectangle query. 245 | * It also contains a filter {@link com.dashlabs.dash.geo.model.filters.GeoFilter} that needs to be applied to the results of the query 246 | * to ensure that everything is in the bounding box of the queried rectangle. 247 | * This is needed because queries are fired for every cell that intersects with the rectangle's bounding box. 248 | * 249 | * @param queryRequest the request that needs to be decorated with geo attributes 250 | * @param minLatitude the latitude of the min point of the rectangle 251 | * @param minLongitude the longitude of the min point of the rectangle 252 | * @param maxLatitude the latitude of the max point of the rectangle 253 | * @param maxLongitude the longitude of the max point of the rectangle 254 | * @param geoIndexName name of the global secondary index for geo spatial querying 255 | * @param geoHashKeyColumn name of the column that stores the item's geoHashKey. This column is used as a hash key of the global secondary index 256 | * @param geoHashColumn name of the column that stores the item's geohash. This column is used as a range key in the global secondary index 257 | * @param geoHashKeyLength the length of the geohashKey. GeoHashKey is a substring of the item's geohash 258 | * @param compositeKeyValue the value of the column that is used in the construction of the composite hash key(geoHashKey + someOtherColumnValue). 259 | * This is needed when constructing queries that need a composite hash key. 260 | * For eg. Fetch an item where lat/long is 23.78787, -70.6767 AND category = 'restaurants' 261 | * @return the wrapper containing the generated queries and the geo filter 262 | */ 263 | public GeoQueryRequest rectangleQuery(QueryRequest queryRequest, double minLatitude, double minLongitude, double maxLatitude, 264 | double maxLongitude, String geoIndexName, String geoHashKeyColumn, 265 | String geoHashColumn, int geoHashKeyLength, Optional compositeKeyValue) { 266 | GeoConfig config = new GeoConfig.Builder().geoHashColumn(geoHashColumn).geoHashKeyColumn(geoHashKeyColumn).geoHashKeyLength( 267 | geoHashKeyLength).geoIndexName(geoIndexName).build(); 268 | return rectangleQuery(queryRequest, minLatitude, minLongitude, maxLatitude, maxLongitude, config, compositeKeyValue); 269 | } 270 | 271 | /** 272 | * Checks the values of the geo config 273 | * 274 | * @param geoIndexName name of the global secondary index for geo spatial querying 275 | * @param geoHashKeyColumn name of the column that stores the item's geoHashKey. This column is used as a hash key of the global secondary index 276 | * @param geoHashColumn name of the column that stores the item's geohash. This column is used as a range key in the global secondary index 277 | * @param geoHashKeyLength the length of the geohashKey. GeoHashKey is a substring of the item's geohash 278 | */ 279 | private void checkConfigParams(String geoIndexName, String geoHashKeyColumn, String geoHashColumn, int geoHashKeyLength) { 280 | checkArgument((geoIndexName != null && geoIndexName.length() > 0), "geoIndexName cannot be empty: %s", geoIndexName); 281 | checkArgument((geoHashKeyColumn != null && geoHashKeyColumn.length() > 0), "geoHashKeyColumn cannot be empty: %s", 282 | geoHashKeyColumn); 283 | checkArgument((geoHashColumn != null && geoHashColumn.length() > 0), "geoHashColumn cannot be empty: %s", geoHashColumn); 284 | checkArgument(geoHashKeyLength > 0, "geoHashKeyLength must be a positive number: %s", String.valueOf(geoHashKeyLength)); 285 | } 286 | } 287 | -------------------------------------------------------------------------------- /src/main/java/com/amazonaws/geo/GeoConfig.java: -------------------------------------------------------------------------------- 1 | package com.amazonaws.geo; 2 | 3 | import com.google.common.base.Optional; 4 | 5 | /** 6 | * Created by mpuri on 3/24/14 7 | */ 8 | public class GeoConfig { 9 | 10 | /** 11 | * The index name of the global secondary index that exists on a table for GeoSpatial querying. 12 | * It comprises of the geoHashKeyColumn(hashKey) and geoHashColumn(range key). 13 | */ 14 | private final String geoIndexName; 15 | 16 | /** 17 | * The hashKey of the global secondary index used for GeoSpatial querying. 18 | * It's value is derived from the geoHashColumn in conjunction with the geoHashKeyLength 19 | */ 20 | private final String geoHashKeyColumn; 21 | 22 | /** 23 | * The column containing the geoHash for a lat/long pair. 24 | * It is mapped as the range key in the global secondary index used for GeoSpatial querying. 25 | */ 26 | private final String geoHashColumn; 27 | 28 | /** 29 | * The size of the hashKey used in the global secondary index used for GeoSpatial querying. 30 | */ 31 | private final int geoHashKeyLength; 32 | 33 | /** 34 | * An optional decorator used to construct the geoHashKey using a composite key. 35 | */ 36 | private final Optional hashKeyDecorator; 37 | 38 | /** 39 | * An optional composite key column. 40 | */ 41 | private final Optional compositeHashKeyColumn; 42 | 43 | public GeoConfig(String geoIndexName, String geoHashKeyColumn, String geoHashColumn, int geoHashKeyLength, Optional hashKeyDecorator, Optional compositeHashKeyColumn) { 44 | this.geoIndexName = geoIndexName; 45 | this.geoHashKeyColumn = geoHashKeyColumn; 46 | this.geoHashColumn = geoHashColumn; 47 | this.geoHashKeyLength = geoHashKeyLength; 48 | this.hashKeyDecorator = hashKeyDecorator == null ? Optional.absent() : hashKeyDecorator; 49 | this.compositeHashKeyColumn = compositeHashKeyColumn == null ? Optional.absent() : compositeHashKeyColumn; 50 | } 51 | 52 | public String getGeoIndexName() { 53 | return geoIndexName; 54 | } 55 | 56 | public String getGeoHashKeyColumn() { 57 | return geoHashKeyColumn; 58 | } 59 | 60 | public String getGeoHashColumn() { 61 | return geoHashColumn; 62 | } 63 | 64 | public int getGeoHashKeyLength() { 65 | return geoHashKeyLength; 66 | } 67 | 68 | public Optional getHashKeyDecorator() { 69 | return hashKeyDecorator; 70 | } 71 | 72 | public Optional getCompositeHashKeyColumn() { 73 | return compositeHashKeyColumn; 74 | } 75 | 76 | /** 77 | * Builder to help with the construction of a GeoConfig 78 | */ 79 | public static class Builder { 80 | private String geoIndexName; 81 | private String geoHashKeyColumn; 82 | private String geoHashColumn; 83 | private int geoHashKeyLength; 84 | private Optional hashKeyDecorator; 85 | private Optional compositeHashKeyColumn; 86 | 87 | public Builder() { 88 | 89 | } 90 | 91 | public Builder geoIndexName(String geoIndexName) { 92 | this.geoIndexName = geoIndexName; 93 | return this; 94 | } 95 | 96 | public Builder geoHashKeyColumn(String geoHashKeyColumn) { 97 | this.geoHashKeyColumn = geoHashKeyColumn; 98 | return this; 99 | } 100 | 101 | public Builder geoHashColumn(String geoHashColumn) { 102 | this.geoHashColumn = geoHashColumn; 103 | return this; 104 | } 105 | 106 | public Builder geoHashKeyLength(int geoHashKeyLength) { 107 | this.geoHashKeyLength = geoHashKeyLength; 108 | return this; 109 | } 110 | 111 | public Builder hashKeyDecorator(Optional value) { 112 | this.hashKeyDecorator = value; 113 | return this; 114 | } 115 | 116 | public Builder compositeHashKeyColumn(Optional value) { 117 | this.compositeHashKeyColumn = value; 118 | return this; 119 | } 120 | 121 | public GeoConfig build() { 122 | return new GeoConfig(this.geoIndexName, this.geoHashKeyColumn, this.geoHashColumn, this.geoHashKeyLength, this.hashKeyDecorator, this.compositeHashKeyColumn); 123 | } 124 | 125 | } 126 | 127 | @Override 128 | public boolean equals(Object o) { 129 | if (this == o) { 130 | return true; 131 | } 132 | if (o == null || getClass() != o.getClass()) { 133 | return false; 134 | } 135 | 136 | GeoConfig geoConfig = (GeoConfig) o; 137 | 138 | if (geoHashKeyLength != geoConfig.geoHashKeyLength) { 139 | return false; 140 | } 141 | if (geoHashColumn != null ? !geoHashColumn.equals(geoConfig.geoHashColumn) : geoConfig.geoHashColumn != null) { 142 | return false; 143 | } 144 | if (geoHashKeyColumn != null ? !geoHashKeyColumn.equals(geoConfig.geoHashKeyColumn) : geoConfig.geoHashKeyColumn != null) { 145 | return false; 146 | } 147 | if (geoIndexName != null ? !geoIndexName.equals(geoConfig.geoIndexName) : geoConfig.geoIndexName != null) { 148 | return false; 149 | } 150 | if (compositeHashKeyColumn != null ? !compositeHashKeyColumn.equals(geoConfig.compositeHashKeyColumn) : geoConfig.compositeHashKeyColumn != null) { 151 | return false; 152 | } 153 | 154 | return true; 155 | } 156 | 157 | @Override 158 | public int hashCode() { 159 | int result = geoIndexName != null ? geoIndexName.hashCode() : 0; 160 | result = 31 * result + (geoHashKeyColumn != null ? geoHashKeyColumn.hashCode() : 0); 161 | result = 31 * result + (geoHashColumn != null ? geoHashColumn.hashCode() : 0); 162 | result = 31 * result + geoHashKeyLength; 163 | result = 31 * result + (compositeHashKeyColumn != null ? compositeHashKeyColumn.hashCode() : 0); 164 | return result; 165 | } 166 | } 167 | -------------------------------------------------------------------------------- /src/main/java/com/amazonaws/geo/GeoQueryHelper.java: -------------------------------------------------------------------------------- 1 | package com.amazonaws.geo; 2 | 3 | import com.dashlabs.dash.geo.AbstractGeoQueryHelper; 4 | import com.dashlabs.dash.geo.model.GeohashRange; 5 | import com.dashlabs.dash.geo.s2.internal.S2Manager; 6 | import com.amazonaws.services.dynamodbv2.model.AttributeValue; 7 | import com.amazonaws.services.dynamodbv2.model.ComparisonOperator; 8 | import com.amazonaws.services.dynamodbv2.model.Condition; 9 | import com.amazonaws.services.dynamodbv2.model.QueryRequest; 10 | import com.google.common.base.Optional; 11 | import com.google.common.collect.ImmutableList; 12 | import com.google.common.geometry.S2LatLngRect; 13 | 14 | import java.util.ArrayList; 15 | import java.util.HashMap; 16 | import java.util.List; 17 | import java.util.Map; 18 | 19 | /** 20 | * Created by mpuri on 3/25/14 21 | */ 22 | public class GeoQueryHelper extends AbstractGeoQueryHelper { 23 | 24 | public GeoQueryHelper(S2Manager s2Manager) { 25 | super(s2Manager); 26 | } 27 | 28 | /** 29 | * For the given QueryRequest query and the boundingBox, this method creates a collection of queries 30 | * that are decorated with geo attributes to enable geo-spatial querying. 31 | * 32 | * @param query the original query request 33 | * @param boundingBox the bounding lat long rectangle of the geo query 34 | * @param config the config containing caller's geo config, example index name, etc. 35 | * @param compositeKeyValue the value of the column that is used in the construction of the composite hash key(geoHashKey + someOtherColumnValue). 36 | * This is needed when constructing queries that need a composite hash key. 37 | * For eg. Fetch an item where lat/long is 23.78787, -70.6767 AND category = 'restaurants' 38 | * @return queryRequests an immutable collection of QueryRequest that are now "geo enabled" 39 | */ 40 | public List generateGeoQueries(QueryRequest query, S2LatLngRect boundingBox, GeoConfig config, Optional compositeKeyValue) { 41 | List outerRanges = getGeoHashRanges(boundingBox); 42 | List queryRequests = new ArrayList(outerRanges.size()); 43 | //Create multiple queries based on the geo ranges derived from the bounding box 44 | for (GeohashRange outerRange : outerRanges) { 45 | List geohashRanges = outerRange.trySplit(config.getGeoHashKeyLength(), s2Manager); 46 | for (GeohashRange range : geohashRanges) { 47 | //Make a copy of the query request to retain original query attributes like table name, etc. 48 | QueryRequest queryRequest = copyQueryRequest(query); 49 | 50 | //generate the hash key for the global secondary index 51 | long geohashKey = s2Manager.generateHashKey(range.getRangeMin(), config.getGeoHashKeyLength()); 52 | Map keyConditions = new HashMap(2, 1.0f); 53 | 54 | //Construct the hashKey condition 55 | Condition geoHashKeyCondition; 56 | if (config.getHashKeyDecorator().isPresent() && compositeKeyValue.isPresent()) { 57 | String compositeHashKey = config.getHashKeyDecorator().get().decorate(compositeKeyValue.get(), geohashKey); 58 | geoHashKeyCondition = new Condition().withComparisonOperator(ComparisonOperator.EQ) 59 | .withAttributeValueList(new AttributeValue().withS(compositeHashKey)); 60 | } else { 61 | geoHashKeyCondition = new Condition().withComparisonOperator(ComparisonOperator.EQ) 62 | .withAttributeValueList(new AttributeValue().withN(String.valueOf(geohashKey))); 63 | } 64 | keyConditions.put(config.getGeoHashKeyColumn(), geoHashKeyCondition); 65 | 66 | //generate the geo hash range 67 | AttributeValue minRange = new AttributeValue().withN(Long.toString(range.getRangeMin())); 68 | AttributeValue maxRange = new AttributeValue().withN(Long.toString(range.getRangeMax())); 69 | 70 | Condition geoHashCondition = new Condition().withComparisonOperator(ComparisonOperator.BETWEEN) 71 | .withAttributeValueList(minRange, maxRange); 72 | keyConditions.put(config.getGeoHashColumn(), geoHashCondition); 73 | 74 | queryRequest.withKeyConditions(keyConditions) 75 | .withIndexName(config.getGeoIndexName()); 76 | queryRequests.add(queryRequest); 77 | } 78 | } 79 | return ImmutableList.copyOf(queryRequests); 80 | } 81 | 82 | /** 83 | * Creates a copy of the provided QueryRequest queryRequest 84 | * 85 | * @param queryRequest 86 | * @return a new 87 | */ 88 | private QueryRequest copyQueryRequest(QueryRequest queryRequest) { 89 | QueryRequest copiedQueryRequest = new QueryRequest().withAttributesToGet(queryRequest.getAttributesToGet()) 90 | .withConsistentRead(queryRequest.getConsistentRead()) 91 | .withExclusiveStartKey(queryRequest.getExclusiveStartKey()) 92 | .withIndexName(queryRequest.getIndexName()) 93 | .withKeyConditions(queryRequest.getKeyConditions()) 94 | .withLimit(queryRequest.getLimit()) 95 | .withReturnConsumedCapacity(queryRequest.getReturnConsumedCapacity()) 96 | .withScanIndexForward(queryRequest.getScanIndexForward()) 97 | .withSelect(queryRequest.getSelect()) 98 | .withAttributesToGet(queryRequest.getAttributesToGet()) 99 | .withTableName(queryRequest.getTableName()); 100 | 101 | return copiedQueryRequest; 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /src/main/java/com/amazonaws/geo/HashKeyDecorator.java: -------------------------------------------------------------------------------- 1 | package com.amazonaws.geo; 2 | 3 | /** 4 | * Created by mpuri on 4/14/14. 5 | * This decorator is used to create a composite geoHashKey. You can use a composite geoHashKey to fire 6 | * geo queries that also take into consideration another column. 7 | * For eg. "Fetch an item where lat/long is 23.78787, -70.6767 AND category = 'restaurants'. 8 | */ 9 | public interface HashKeyDecorator { 10 | 11 | /** 12 | * Creates a composite hashKey used for geo querying. 13 | * @param columnValue the value of the column that needs to be part of the hashKey 14 | * @param geoHashKey the geoHashKey of the item 15 | * @return a string containing the geoHashKey and the additional column value 16 | */ 17 | String decorate(String columnValue, long geoHashKey); 18 | 19 | } 20 | -------------------------------------------------------------------------------- /src/main/java/com/amazonaws/geo/model/GeoQueryRequest.java: -------------------------------------------------------------------------------- 1 | package com.amazonaws.geo.model; 2 | 3 | import com.amazonaws.services.dynamodbv2.model.AttributeValue; 4 | import com.amazonaws.services.dynamodbv2.model.QueryRequest; 5 | import com.dashlabs.dash.geo.model.filters.GeoFilter; 6 | 7 | import java.util.List; 8 | import java.util.Map; 9 | 10 | /** 11 | * Created by mpuri on 3/25/14. 12 | * A wrapper that encapsulates the collection of queries that are generated for a radius or a rectangle query 13 | * and the filter that has to be applied to the query results. 14 | */ 15 | public class GeoQueryRequest { 16 | 17 | private final List queryRequests; 18 | 19 | private final GeoFilter> resultFilter; 20 | 21 | public GeoQueryRequest(List queryRequests, GeoFilter> resultFilter) { 22 | this.queryRequests = queryRequests; 23 | this.resultFilter = resultFilter; 24 | } 25 | 26 | public List getQueryRequests() { 27 | return queryRequests; 28 | } 29 | 30 | public GeoFilter> getResultFilter() { 31 | return resultFilter; 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/com/amazonaws/geo/model/filters/GeoFilters.java: -------------------------------------------------------------------------------- 1 | package com.amazonaws.geo.model.filters; 2 | 3 | import com.amazonaws.services.dynamodbv2.model.AttributeValue; 4 | import com.dashlabs.dash.geo.model.filters.GeoDataExtractor; 5 | import com.dashlabs.dash.geo.model.filters.GeoFilter; 6 | import com.dashlabs.dash.geo.model.filters.RadiusGeoFilter; 7 | import com.dashlabs.dash.geo.model.filters.RectangleGeoFilter; 8 | import com.google.common.geometry.S2LatLng; 9 | import com.google.common.geometry.S2LatLngRect; 10 | 11 | import java.util.Map; 12 | import java.util.Optional; 13 | 14 | /** 15 | * Created by mpuri on 3/26/14. 16 | * Factory methods for {@link GeoFilter}. 17 | */ 18 | public class GeoFilters { 19 | 20 | private static final GeoDataExtractor> EXTRACTOR = new GeoDataExtractor>() { 21 | @Override public Optional extractLatitude(Map item) { 22 | if ((item.get(GeoFilter.LATITUDE_FIELD) != null) && (item.get(GeoFilter.LATITUDE_FIELD).getN() != null)) { 23 | return Optional.of(Double.valueOf(item.get(GeoFilter.LATITUDE_FIELD).getN())); 24 | } 25 | return Optional.empty(); 26 | } 27 | 28 | @Override public Optional extractLongitude(Map item) { 29 | if ((item.get(GeoFilter.LONGITUDE_FIELD) != null) && (item.get(GeoFilter.LONGITUDE_FIELD).getN() != null)) { 30 | return Optional.of(Double.valueOf(item.get(GeoFilter.LONGITUDE_FIELD).getN())); 31 | } 32 | return Optional.empty(); 33 | } 34 | }; 35 | 36 | /** 37 | * Factory method to create a filter used by radius queries. 38 | * 39 | * @param centerLatLng the lat/long of the center of the filter's radius 40 | * @param radiusInMeter the radius of the filter in metres 41 | * @return a new instance of the {@link RadiusGeoFilter} 42 | */ 43 | public static GeoFilter> newRadiusFilter(S2LatLng centerLatLng, double radiusInMeter) { 44 | return com.dashlabs.dash.geo.model.filters.GeoFilters.newRadiusFilter(EXTRACTOR, centerLatLng, radiusInMeter); 45 | } 46 | 47 | /** 48 | * Factory method to create a filter used by rectangle queries 49 | * 50 | * @param latLngRect the bounding box for the filter 51 | * @return a new instance of the {@link RectangleGeoFilter} 52 | */ 53 | public static GeoFilter> newRectangleFilter(S2LatLngRect latLngRect) { 54 | return com.dashlabs.dash.geo.model.filters.GeoFilters.newRectangleFilter(EXTRACTOR, latLngRect); 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /src/main/java/com/amazonaws/geo/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2010-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | /** 16 | * Geo Library for Amazon DynamoDB 17 | *

18 | * This server library will enable you to create, retrieve, and query for geo-spatial data records in DynamoDB. Geo data 19 | * is popular in mobile apps, and along with this library we have sample mobile projects for iOS which 20 | * demonstrate usage. 21 | *

22 | * Geo-spatial data in a nutshell 23 | *

24 | * A Geo-spatial data record is simply one which is tagged with a latitude and a longitude, which correspond to that 25 | * record's "place" on the planet Earth. Many mobile apps which run on smart phones generate geo data records by 26 | * accessing the GPS to determine the current location, then pass the lat/lng pair to a data storage layer for retrieval 27 | * in the future. Mobile apps also commonly access geo data records which are "nearby". This is done by performing a 28 | * query for all the geo data records which are within a radius or a bounding box around the current location of the 29 | * device. 30 | *

31 | *

32 | * Storing, retrieving, and querying for data records which have a physical location has always been possible with 33 | * DynamoDB, but it required careful design of hash keys, ranges, and indexes, as well as associated code for 34 | * calculating a point's position "inside or outside" a given grid which overlays the globe. Our new library makes it 35 | * very easy. 36 | *

37 | * Major components: 38 | *
    39 | *
  • Geo Library for Amazon DynamoDB: Java library which provides a high-level interface to DynamoDB-backed data 40 | * storage of geo objects. This library can be added to your existing server backend. For more help getting started, we 41 | * include a reference sample Java server for AWS ElasticBeanstalk.
  • 42 | *
  • 43 | * DynamoDB tables: you create and configure these tables to store your geo records. The library takes care of 44 | * determining the proper hash keys, range keys, and indexes. These keys and indexes are how queries can "find" the 45 | * records which are within a physical distance of a given starting point.
  • 46 | *
  • 47 | * Client code: interacting with the server library can occur any way you prefer. We have included reference samples of 48 | * mobile apps for iOS and Android which demonstrate how to gather the lat/lng from the device and send it to the 49 | * library, as well as how to query for geo records and display them on a map view.
  • 50 | *
51 | * How does a query work? 52 | *

53 | * We overlay a virtual "grid" over the planet earth. Each grid cell has a corresponding "address", derived by location. 54 | * When geo points are inserted to DynamoDB, a "geohash" is constructed, which locates the data record into the correct 55 | * grid cell. Geohashes also attempt to preserve the proximity of nearby points. Using local secondary indexes, the 56 | * geohash is stored in DynamoDB along with the data record. 57 | *

58 | *

59 | * For querying, you provide either a center point lat/lng and a radial distance, or the coordinates of a bounding box. 60 | * The library uses this input to determine which grid cells are "candidates" for returning geo records from DynamoDB. 61 | * Those cells are queried by geohash - again, a local secondary index on the DynamoDB table - to return the candidate 62 | * geo records. Some cells contain lots of points, some contain none, depending on what points were originally stored in 63 | * the table. Next, the library post-processes those records, filtering out the ones which are outside of the 64 | * originally-provided input (the bounding box or radius). The final list of matching geo records are returned to the 65 | * client. 66 | *

67 | * 68 | * @since 1.0 69 | * @version 1.0 70 | * */ 71 | package com.amazonaws.geo; -------------------------------------------------------------------------------- /src/main/java/com/amazonaws/geo/s2/internal/GeoQueryClient.java: -------------------------------------------------------------------------------- 1 | package com.amazonaws.geo.s2.internal; 2 | 3 | import com.amazonaws.geo.model.GeoQueryRequest; 4 | import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient; 5 | import com.amazonaws.services.dynamodbv2.model.AttributeValue; 6 | import com.amazonaws.services.dynamodbv2.model.QueryRequest; 7 | import com.amazonaws.services.dynamodbv2.model.QueryResult; 8 | import com.dashlabs.dash.geo.model.filters.GeoFilter; 9 | import com.google.common.collect.ImmutableList; 10 | 11 | import java.util.ArrayList; 12 | import java.util.List; 13 | import java.util.Map; 14 | import java.util.concurrent.Callable; 15 | import java.util.concurrent.ExecutionException; 16 | import java.util.concurrent.ExecutorService; 17 | import java.util.concurrent.Future; 18 | 19 | /** 20 | * Created by mpuri on 3/28/14 21 | */ 22 | public class GeoQueryClient { 23 | 24 | /** 25 | * The db client to use when executing the queries 26 | */ 27 | private final AmazonDynamoDBClient dbClient; 28 | 29 | /** 30 | * The executor service to use to manage the queries workload 31 | */ 32 | private final ExecutorService executorService; 33 | 34 | public GeoQueryClient(AmazonDynamoDBClient dbClient, ExecutorService executorService) { 35 | this.dbClient = dbClient; 36 | this.executorService = executorService; 37 | } 38 | 39 | /** 40 | * A convenience method that executes the queryRequests and applies the resultFilter to the query results. 41 | * 42 | * @return an immutable collection of filtered items 43 | */ 44 | public List> execute(final GeoQueryRequest geoQueryRequest) 45 | throws InterruptedException, ExecutionException { 46 | final List> results = new ArrayList>(); 47 | List>>> futures; 48 | final List>>> queryCallables = 49 | new ArrayList>>>(geoQueryRequest.getQueryRequests().size()); 50 | for (final QueryRequest query : geoQueryRequest.getQueryRequests()) { 51 | queryCallables.add(new Callable>>() { 52 | @Override public List> call() throws Exception { 53 | return executeQuery(query, geoQueryRequest.getResultFilter()); 54 | } 55 | }); 56 | } 57 | futures = executorService.invokeAll(queryCallables); 58 | if (futures != null) { 59 | for (Future>> future : futures) { 60 | results.addAll(future.get()); 61 | } 62 | } 63 | return ImmutableList.copyOf(results); 64 | } 65 | 66 | /** 67 | * Executes the query using the provided db client. The geo filter is applied to the results of the query. 68 | * 69 | * @param queryRequest the query to execute 70 | * @return a collection of filtered result items 71 | */ 72 | private List> executeQuery(QueryRequest queryRequest, GeoFilter> resultFilter) { 73 | QueryResult queryResult; 74 | List> resultItems = new ArrayList>(); 75 | do { 76 | queryResult = dbClient.query(queryRequest); 77 | List> items = queryResult.getItems(); 78 | // filter the results using the geo filter 79 | List> filteredItems = resultFilter.filter(items); 80 | resultItems.addAll(filteredItems); 81 | queryRequest = queryRequest.withExclusiveStartKey(queryResult.getLastEvaluatedKey()); 82 | } while ((queryResult.getLastEvaluatedKey() != null)); 83 | 84 | return resultItems; 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /src/test/java/com/amazonaws/geo/GeoQueryClientTest.java: -------------------------------------------------------------------------------- 1 | package com.amazonaws.geo; 2 | 3 | import com.amazonaws.geo.model.GeoQueryRequest; 4 | import com.amazonaws.geo.s2.internal.GeoQueryClient; 5 | import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient; 6 | import com.amazonaws.services.dynamodbv2.model.AttributeValue; 7 | import com.amazonaws.services.dynamodbv2.model.QueryRequest; 8 | import com.amazonaws.services.dynamodbv2.model.QueryResult; 9 | import com.dashlabs.dash.geo.model.filters.GeoFilter; 10 | import org.junit.Test; 11 | 12 | import java.util.*; 13 | import java.util.concurrent.ExecutionException; 14 | import java.util.concurrent.ExecutorService; 15 | import java.util.concurrent.Executors; 16 | 17 | import static org.junit.Assert.assertEquals; 18 | import static org.junit.Assert.assertNotNull; 19 | import static org.junit.Assert.fail; 20 | import static org.mockito.Mockito.mock; 21 | import static org.mockito.Mockito.when; 22 | 23 | /** 24 | * Created by mpuri on 3/26/14 25 | */ 26 | public class GeoQueryClientTest { 27 | 28 | @Test @SuppressWarnings("unchecked") 29 | public void execute() { 30 | AmazonDynamoDBClient dbClient = mock(AmazonDynamoDBClient.class); 31 | GeoFilter> geoFilter = mock(GeoFilter.class); 32 | ExecutorService executorService = Executors.newFixedThreadPool(2); 33 | GeoQueryClient geoQueryClient = new GeoQueryClient(dbClient,executorService); 34 | 35 | //Mock queries that get fired by the execute method 36 | List queryRequests = new ArrayList(); 37 | QueryRequest query1 = new QueryRequest().withLimit(5); 38 | QueryRequest query2 = new QueryRequest().withLimit(10); 39 | queryRequests.add(query1); 40 | queryRequests.add(query2); 41 | 42 | //Mock results of the first query 43 | QueryResult result1 = new QueryResult(); 44 | List> resultItems1 = new ArrayList>(); 45 | Map item1 = new HashMap(); 46 | item1.put("title", new AttributeValue().withS("Milk Bar")); 47 | item1.put("tag", new AttributeValue().withS("cafe:breakfast:american")); 48 | 49 | Map item2 = new HashMap(); 50 | item2.put("title", new AttributeValue().withS("Chuko")); 51 | item2.put("tag", new AttributeValue().withS("restaurant:noodles:japanese")); 52 | resultItems1.add(item1); 53 | resultItems1.add(item2); 54 | result1.setItems(resultItems1); 55 | 56 | //Mock results of the second query 57 | QueryResult result2 = new QueryResult(); 58 | List> resultItems2 = new ArrayList>(); 59 | Map item3 = new HashMap(); 60 | item3.put("title", new AttributeValue().withS("Al Di La")); 61 | item3.put("tag", new AttributeValue().withS("restaurant:italian")); 62 | 63 | Map item4 = new HashMap(); 64 | item4.put("title", new AttributeValue().withS("Blue Print")); 65 | item4.put("tag", new AttributeValue().withS("bar:cocktails")); 66 | resultItems2.add(item3); 67 | resultItems2.add(item4); 68 | result2.setItems(resultItems2); 69 | 70 | when(dbClient.query(query1)).thenReturn(result1); 71 | when(dbClient.query(query2)).thenReturn(result2); 72 | when(geoFilter.filter(resultItems1)).thenReturn(resultItems1); 73 | List> filteredList = new ArrayList>(); 74 | filteredList.add(resultItems2.get(0)); 75 | when(geoFilter.filter(resultItems2)).thenReturn(filteredList); 76 | 77 | GeoQueryRequest geoQueryRequest = new GeoQueryRequest(queryRequests, geoFilter); 78 | try { 79 | List> results = geoQueryClient.execute(geoQueryRequest); 80 | assertNotNull(results); 81 | assertEquals(results.size(), 3); 82 | } catch (InterruptedException | ExecutionException ie) { 83 | fail("error occurred while executing the queries"); 84 | } finally { 85 | executorService.shutdown(); 86 | } 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /src/test/java/com/amazonaws/geo/GeoTest.java: -------------------------------------------------------------------------------- 1 | package com.amazonaws.geo; 2 | 3 | import com.amazonaws.geo.model.GeoQueryRequest; 4 | import com.amazonaws.services.dynamodbv2.model.*; 5 | import com.dashlabs.dash.geo.s2.internal.S2Manager; 6 | import com.google.common.base.Optional; 7 | import com.google.common.geometry.S2LatLng; 8 | import com.google.common.geometry.S2LatLngRect; 9 | import org.junit.Test; 10 | 11 | import java.util.ArrayList; 12 | import java.util.HashMap; 13 | import java.util.List; 14 | import java.util.Map; 15 | 16 | import static org.junit.Assert.*; 17 | import static org.mockito.Mockito.*; 18 | 19 | /** 20 | * Created by mpuri on 3/26/14 21 | */ 22 | public class GeoTest { 23 | 24 | @Test 25 | public void putItemRequestInvalidFields() { 26 | Geo geo = new Geo(); 27 | try { 28 | List configs = new ArrayList(); 29 | configs.add(new GeoConfig(null, null, null, 0, Optional.absent(), null)); 30 | geo.putItemRequest(new PutItemRequest(), 0.0, 0.0, configs); 31 | fail("Should have failed as there are invalid fields"); 32 | } catch (IllegalArgumentException e) { 33 | //expected 34 | } 35 | } 36 | 37 | @Test 38 | public void putItemRequest() { 39 | GeoQueryHelper geoQueryHelper = mock(GeoQueryHelper.class); 40 | S2Manager s2Manager = mock(S2Manager.class); 41 | Geo geo = new Geo(s2Manager, geoQueryHelper); 42 | double lat = 5.0; 43 | double longitude = -5.5; 44 | long geohash = System.currentTimeMillis(); 45 | long geohashKey = 12345; 46 | List configs = new ArrayList(); 47 | GeoConfig config = createTestConfig(false, null); 48 | configs.add(config); 49 | String tableName = "TableWithSomeData"; 50 | Map populatedItem = new HashMap(); 51 | populatedItem.put("title", new AttributeValue().withS("Ippudo")); 52 | PutItemRequest request = new PutItemRequest().withTableName(tableName).withItem(populatedItem); 53 | when(s2Manager.generateGeohash(lat, longitude)).thenReturn(geohash); 54 | when(s2Manager.generateHashKey(geohash, config.getGeoHashKeyLength())).thenReturn(geohashKey); 55 | 56 | PutItemRequest withGeoProperties = geo.putItemRequest(request, lat, longitude, configs); 57 | assertNotNull(withGeoProperties); 58 | assertEquals(request.getTableName(), withGeoProperties.getTableName()); 59 | assertEquals(request.getItem().get("title"), withGeoProperties.getItem().get("title")); 60 | assertEquals(withGeoProperties.getItem().get(config.getGeoHashColumn()).getN(), String.valueOf(geohash)); 61 | assertEquals(withGeoProperties.getItem().get(config.getGeoHashKeyColumn()).getN(), String.valueOf(geohashKey)); 62 | verify(s2Manager, times(1)).generateGeohash(lat, longitude); 63 | verify(s2Manager, times(1)).generateHashKey(geohash, config.getGeoHashKeyLength()); 64 | verifyNoMoreInteractions(s2Manager); 65 | } 66 | 67 | @Test 68 | public void putItemRequestWithCompositeColumn() { 69 | GeoQueryHelper geoQueryHelper = mock(GeoQueryHelper.class); 70 | S2Manager s2Manager = mock(S2Manager.class); 71 | Geo geo = new Geo(s2Manager, geoQueryHelper); 72 | double lat = 5.0; 73 | double longitude = -5.5; 74 | long geohash = System.currentTimeMillis(); 75 | long geohashKey = 12345; 76 | List configs = new ArrayList(); 77 | GeoConfig config = createTestConfig(true, "venueCategory"); 78 | configs.add(config); 79 | String tableName = "TableWithSomeData"; 80 | Map populatedItem = new HashMap(); 81 | populatedItem.put("title", new AttributeValue().withS("Ippudo")); 82 | populatedItem.put("venueCategory", new AttributeValue().withS("restaurant")); 83 | PutItemRequest request = new PutItemRequest().withTableName(tableName).withItem(populatedItem); 84 | when(s2Manager.generateGeohash(lat, longitude)).thenReturn(geohash); 85 | when(s2Manager.generateHashKey(geohash, config.getGeoHashKeyLength())).thenReturn(geohashKey); 86 | 87 | PutItemRequest withGeoProperties = geo.putItemRequest(request, lat, longitude, configs); 88 | assertNotNull(withGeoProperties); 89 | assertEquals(request.getTableName(), withGeoProperties.getTableName()); 90 | assertEquals(request.getItem().get("title"), withGeoProperties.getItem().get("title")); 91 | assertEquals(withGeoProperties.getItem().get(config.getGeoHashColumn()).getN(), String.valueOf(geohash)); 92 | assertEquals(withGeoProperties.getItem().get(config.getGeoHashKeyColumn()).getS(), new DefaultHashKeyDecorator().decorate("restaurant", geohashKey)); 93 | verify(s2Manager, times(1)).generateGeohash(lat, longitude); 94 | verify(s2Manager, times(1)).generateHashKey(geohash, config.getGeoHashKeyLength()); 95 | verifyNoMoreInteractions(s2Manager); 96 | 97 | reset(s2Manager); 98 | 99 | // test where the composite value is null, the composite GSI should not be added to the put-item request 100 | populatedItem.clear(); 101 | populatedItem.put("title", new AttributeValue().withS("Ippudo")); 102 | request = new PutItemRequest().withTableName(tableName).withItem(populatedItem); 103 | when(s2Manager.generateGeohash(lat, longitude)).thenReturn(geohash); 104 | when(s2Manager.generateHashKey(geohash, config.getGeoHashKeyLength())).thenReturn(geohashKey); 105 | 106 | withGeoProperties = geo.putItemRequest(request, lat, longitude, configs); 107 | assertNotNull(withGeoProperties); 108 | assertEquals(request.getTableName(), withGeoProperties.getTableName()); 109 | assertEquals(request.getItem().get("title"), withGeoProperties.getItem().get("title")); 110 | assertEquals(withGeoProperties.getItem().get(config.getGeoHashColumn()).getN(), String.valueOf(geohash)); 111 | assertNull(withGeoProperties.getItem().get(config.getGeoHashKeyColumn())); 112 | verify(s2Manager, times(1)).generateGeohash(lat, longitude); 113 | verify(s2Manager, times(1)).generateHashKey(geohash, config.getGeoHashKeyLength()); 114 | verifyNoMoreInteractions(s2Manager); 115 | 116 | } 117 | 118 | @Test 119 | public void getItemQueryInvalidFields() { 120 | Geo geo = new Geo(); 121 | try { 122 | geo.getItemQuery(new QueryRequest(), 0.0, 0.0, null, null, null, 0, Optional.absent()); 123 | fail("Should have failed as there are invalid fields"); 124 | } catch (IllegalArgumentException e) { 125 | //expected 126 | } 127 | } 128 | 129 | @Test 130 | public void getItemQuery() { 131 | GeoQueryHelper geoQueryHelper = mock(GeoQueryHelper.class); 132 | S2Manager s2Manager = mock(S2Manager.class); 133 | Geo geo = new Geo(s2Manager, geoQueryHelper); 134 | double lat = 5.0; 135 | double longitude = -5.5; 136 | long geohash = System.currentTimeMillis(); 137 | long geohashKey = 12345; 138 | Condition expectedGeoHashKeyCondition = new Condition().withComparisonOperator(ComparisonOperator.EQ) 139 | .withAttributeValueList(new AttributeValue().withN(String.valueOf(geohashKey))); 140 | Condition expectedGeoHashCondition = new Condition().withComparisonOperator(ComparisonOperator.EQ) 141 | .withAttributeValueList(new AttributeValue().withN(String.valueOf(geohash))); 142 | 143 | GeoConfig config = createTestConfig(false, null); 144 | String tableName = "TableWithSomeData"; 145 | QueryRequest query = new QueryRequest().withTableName(tableName); 146 | when(s2Manager.generateGeohash(lat, longitude)).thenReturn(geohash); 147 | when(s2Manager.generateHashKey(geohash, config.getGeoHashKeyLength())).thenReturn(geohashKey); 148 | 149 | QueryRequest withGeoProperties = geo.getItemQuery(query, lat, longitude, config, Optional.absent()); 150 | assertNotNull(withGeoProperties); 151 | assertEquals(query.getTableName(), withGeoProperties.getTableName()); 152 | assertEquals(withGeoProperties.getKeyConditions().get(config.getGeoHashKeyColumn()), expectedGeoHashKeyCondition); 153 | assertEquals(withGeoProperties.getKeyConditions().get(config.getGeoHashColumn()), expectedGeoHashCondition); 154 | verify(s2Manager, times(1)).generateGeohash(lat, longitude); 155 | verify(s2Manager, times(1)).generateHashKey(geohash, config.getGeoHashKeyLength()); 156 | verifyNoMoreInteractions(s2Manager); 157 | } 158 | 159 | @Test 160 | public void getItemQueryWithCompositeKey() { 161 | GeoQueryHelper geoQueryHelper = mock(GeoQueryHelper.class); 162 | S2Manager s2Manager = mock(S2Manager.class); 163 | Geo geo = new Geo(s2Manager, geoQueryHelper); 164 | double lat = 5.0; 165 | double longitude = -5.5; 166 | long geohash = System.currentTimeMillis(); 167 | long geohashKey = 12345; 168 | String category = "restaurant"; 169 | Condition expectedGeoHashKeyCondition = new Condition().withComparisonOperator(ComparisonOperator.EQ) 170 | .withAttributeValueList(new AttributeValue().withS(String.format("%s:%d", category, geohashKey))); 171 | Condition expectedGeoHashCondition = new Condition().withComparisonOperator(ComparisonOperator.EQ) 172 | .withAttributeValueList(new AttributeValue().withN(String.valueOf(geohash))); 173 | 174 | GeoConfig config = createTestConfig(true, "category"); 175 | String tableName = "TableWithSomeData"; 176 | QueryRequest query = new QueryRequest().withTableName(tableName); 177 | when(s2Manager.generateGeohash(lat, longitude)).thenReturn(geohash); 178 | when(s2Manager.generateHashKey(geohash, config.getGeoHashKeyLength())).thenReturn(geohashKey); 179 | 180 | QueryRequest withGeoProperties = geo.getItemQuery(query, lat, longitude, config, Optional.of(category)); 181 | assertNotNull(withGeoProperties); 182 | assertEquals(query.getTableName(), withGeoProperties.getTableName()); 183 | assertEquals(withGeoProperties.getKeyConditions().get(config.getGeoHashKeyColumn()), expectedGeoHashKeyCondition); 184 | assertEquals(withGeoProperties.getKeyConditions().get(config.getGeoHashColumn()), expectedGeoHashCondition); 185 | verify(s2Manager, times(1)).generateGeohash(lat, longitude); 186 | verify(s2Manager, times(1)).generateHashKey(geohash, config.getGeoHashKeyLength()); 187 | verifyNoMoreInteractions(s2Manager); 188 | } 189 | 190 | @Test 191 | public void radiusQueryInvalidRadius() { 192 | Geo geo = new Geo(); 193 | try { 194 | geo.radiusQuery(new QueryRequest(), 0.0, 0.0, -5.0, null, null, null, 0, Optional.absent()); 195 | fail("Should have failed as there are invalid fields"); 196 | } catch (IllegalArgumentException e) { 197 | //expected 198 | } 199 | } 200 | 201 | @Test 202 | public void radiusQuery() { 203 | GeoQueryHelper geoQueryHelper = mock(GeoQueryHelper.class); 204 | S2Manager s2Manager = mock(S2Manager.class); 205 | Geo geo = new Geo(s2Manager, geoQueryHelper); 206 | double lat = 5.0; 207 | double longitude = -5.5; 208 | double radius = 50; 209 | String category = "restaurant"; 210 | GeoConfig config = createTestConfig(true, "category"); 211 | String tableName = "TableWithSomeData"; 212 | QueryRequest query = new QueryRequest().withTableName(tableName); 213 | List geoQueries = new ArrayList(); 214 | geoQueries.add(new QueryRequest().withLimit(100)); 215 | S2LatLngRect latLngRect = new S2LatLngRect(S2LatLng.fromDegrees(lat, longitude), S2LatLng.fromDegrees(lat + 10, longitude + 10)); 216 | when(s2Manager.getBoundingBoxForRadiusQuery(lat, longitude, radius)).thenReturn(latLngRect); 217 | when(geoQueryHelper.generateGeoQueries(query, latLngRect, config, Optional.of(category))).thenReturn(geoQueries); 218 | GeoQueryRequest geoQueryRequest = geo.radiusQuery(query, lat, longitude, radius, config, Optional.of(category)); 219 | assertNotNull(geoQueryRequest); 220 | assertNotNull(geoQueryRequest.getResultFilter()); 221 | assertNotNull(geoQueryRequest.getQueryRequests()); 222 | assertEquals(geoQueryRequest.getQueryRequests(), geoQueries); 223 | verify(s2Manager, times(1)).getBoundingBoxForRadiusQuery(lat, longitude, radius); 224 | verify(geoQueryHelper, times(1)).generateGeoQueries(query, latLngRect, config, Optional.of(category)); 225 | verifyNoMoreInteractions(s2Manager, geoQueryHelper); 226 | } 227 | 228 | @Test 229 | public void rectangleQueryInvalidFields() { 230 | Geo geo = new Geo(); 231 | try { 232 | geo.rectangleQuery(new QueryRequest(), 0.0, 0.0, 0.0, 0.0, null, null, null, 0, Optional.absent()); 233 | fail("Should have failed as there are invalid fields"); 234 | } catch (IllegalArgumentException e) { 235 | //expected 236 | } 237 | } 238 | 239 | @Test 240 | public void rectangleQuery() { 241 | GeoQueryHelper geoQueryHelper = mock(GeoQueryHelper.class); 242 | S2Manager s2Manager = mock(S2Manager.class); 243 | Geo geo = new Geo(s2Manager, geoQueryHelper); 244 | double minLat = 5.0; 245 | double minLongitude = -5.5; 246 | double maxLat = 15.0; 247 | double maxLongitude = -25.5; 248 | GeoConfig config = createTestConfig(false, null); 249 | String tableName = "TableWithSomeData"; 250 | QueryRequest query = new QueryRequest().withTableName(tableName); 251 | List geoQueries = new ArrayList(); 252 | geoQueries.add(new QueryRequest().withLimit(100)); 253 | S2LatLngRect latLngRect = new S2LatLngRect(S2LatLng.fromDegrees(minLat, minLongitude), S2LatLng.fromDegrees(maxLat, maxLongitude)); 254 | when(s2Manager.getBoundingBoxForRectangleQuery(minLat, minLongitude, maxLat, maxLongitude)).thenReturn(latLngRect); 255 | when(geoQueryHelper.generateGeoQueries(query, latLngRect, config, Optional.absent())).thenReturn(geoQueries); 256 | GeoQueryRequest geoQueryRequest = geo.rectangleQuery(query, minLat, minLongitude, maxLat, maxLongitude, config, Optional.absent()); 257 | assertNotNull(geoQueryRequest); 258 | assertNotNull(geoQueryRequest.getResultFilter()); 259 | assertNotNull(geoQueryRequest.getQueryRequests()); 260 | assertEquals(geoQueryRequest.getQueryRequests(), geoQueries); 261 | verify(s2Manager, times(1)).getBoundingBoxForRectangleQuery(minLat, minLongitude, maxLat, maxLongitude); 262 | verify(geoQueryHelper, times(1)).generateGeoQueries(query, latLngRect, config, Optional.absent()); 263 | verifyNoMoreInteractions(s2Manager, geoQueryHelper); 264 | } 265 | 266 | private GeoConfig createTestConfig(boolean withKeyDecorator, String compositeColumnName) { 267 | int hashKeyLength = 3; 268 | String geoIndexName = "VenueGeoIndex"; 269 | String geoHashKeyColumn = "geoHashKey"; 270 | String geoHashColumn = "geohash"; 271 | 272 | GeoConfig.Builder builder = new GeoConfig.Builder().geoIndexName(geoIndexName).geoHashKeyLength(hashKeyLength) 273 | .geoHashKeyColumn(geoHashKeyColumn).geoHashColumn(geoHashColumn); 274 | if(withKeyDecorator) { 275 | HashKeyDecorator decorator = new DefaultHashKeyDecorator(); 276 | builder.hashKeyDecorator(Optional.of(decorator)); 277 | builder.compositeHashKeyColumn(Optional.of(compositeColumnName)); 278 | }else{ 279 | builder.hashKeyDecorator(Optional.absent()); 280 | } 281 | return builder.build(); 282 | } 283 | 284 | } 285 | --------------------------------------------------------------------------------