├── .gitignore ├── MANIFEST.in ├── Makefile ├── README.rst ├── geofdw ├── __init__.py ├── _version.py ├── base.py ├── exception.py ├── fdw │ ├── __init__.py │ ├── geocode.py │ ├── geojson.py │ ├── opensky.py │ └── randompoint.py └── utils.py ├── setup.py ├── test.sql └── test ├── __init__.py ├── fdw ├── __init__.py ├── geojson.py └── randompoint.py ├── test_geofdw.py └── test_utils.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | .coverage 3 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-include geofdw *.py 2 | 3 | global-exclude *.py[cod] 4 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: test 2 | 3 | test: 4 | nosetests --verbose --with-cover --cover-erase --cover-package=geofdw 5 | 6 | clean: 7 | find . -name "*.pyc" -print0 | xargs -0 rm -rf 8 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | geofdw 2 | ====== 3 | 4 | ``geofdw`` is a collection of `PostGIS `__-related 5 | `foreign data 6 | wrappers `__ for 7 | `PostgreSQL `__ written in Python using the 8 | `multicorn `__ extension. By using a FDW, you can 9 | access spatial data through Postgres tables without having to import the 10 | data first, which can be useful for dynamic or non-tabular data 11 | available through web services. 12 | 13 | Currently implemented forward data wrappers are: 14 | 15 | - FGeocode: forward geocoding 16 | - RGeocode: reverse geocoding 17 | - GeoJSON: online GeoJSON 18 | - RandomPoint: random point in a bounding box 19 | 20 | ``geofdw`` uses `plpygis `__ to 21 | manipulate PostGIS geometries. 22 | 23 | Examples 24 | -------- 25 | 26 | FGeocode 27 | ~~~~~~~~ 28 | 29 | Create a single server for all forward geocoding tables: 30 | 31 | :: 32 | 33 | # CREATE SERVER fwd_geocode FOREIGN DATA WRAPPER multicorn OPTIONS ( wrapper 'geofdw.FGeocode' ); 34 | 35 | Create two tables, one using the GoogleV3 geocoder and one using 36 | Nominatim: 37 | 38 | :: 39 | 40 | # CREATE FOREIGN TABLE fgc_google ( query TEXT, rank INTEGER, address TEXT, geom GEOMETRY ) SERVER fwd_geocode; 41 | # CREATE FOREIGN TABLE fgc_nominatim ( query TEXT, rank INTEGER, address TEXT, geom GEOMETRY ) SERVER fwd_geocode OPTIONS ( service 'nominatim'); 42 | 43 | Select results from the geocoder matching our query string: 44 | 45 | :: 46 | 47 | SELECT address, ST_AsText(geom) AS geom FROM fgc_google WHERE query = 'canada house' LIMIT 5; 48 | address | geom 49 | ----------------------------------------------------------------------------------------+------------------------------------ 50 | Canada House, London, UK | POINT Z (-0.1291 51.5077 0) 51 | Canada House, Temple Road, Blackrock, Co. Dublin, Ireland | POINT Z (-6.1756563 53.2994401 0) 52 | Canada House, Saint Stephen's Green, Dublin 2, Ireland | POINT Z (-6.2576992 53.335963 0) 53 | Canada House, 29 Hampton Road, Twickenham, Greater London TW2 5QE, UK | POINT Z (-0.3443802 51.441739 0) 54 | Canada House, 272 Field End Road, Ruislip, Greater London HA4 9NA, UK | POINT Z (-0.3973435 51.5752994 0) 55 | 56 | Perform the same query but using the Nominatim geocoder: 57 | 58 | :: 59 | 60 | # SELECT address, ST_AsText(geom) AS geom FROM fgc_nominatim WHERE query = 'canada house' LIMIT 5; 61 | address | geom 62 | -----------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------- 63 | High Commission of Canada, 5, Trafalgar Square, Covent Garden, City of Westminster, London, Greater London, England, SW1Y 5BJ, United Kingdom | POINT Z (-0.129104703393283 51.50782475 0) 64 | Canada House, West 54th Street, Diamond District, Manhattan, New York, NYC, New York, 10019, United States of America | POINT Z (-73.9758789212845 40.7609797 0) 65 | Canada House, Kololo Road, Governments Cantonment, Juba, Central Equatoria, South Sudan | POINT Z (31.5901769 4.8615639 0) 66 | Canada House, Justine Close, Nalumunye, Kazinga, Wakiso, Central 2, Central Region, Uganda | POINT Z (32.4868336484328 0.25676655 0) 67 | בית קנדה, 1, שבי ציון, רובע א', אשדוד, מחוז הדרום, 77452, מדינת ישראל | POINT Z (34.64463585 31.8086878 0) 68 | 69 | Influence the geocoder by restricting the query to a certain bounding 70 | box (in this case, without the hint, Google will only return results in 71 | the USA): 72 | 73 | :: 74 | 75 | # SELECT address, ST_AsText(geom) AS geom FROM fgc_google WHERE query = 'Water St' AND geom && ST_GeomFromEWKT('SRID=4326;POLYGON((50 2, 55 2, 55 -2, 50 -2, 50 2))'); 76 | address | geom 77 | ------------------------------------------------------------------------+----------------------------------- 78 | Water Street, Lavenham, Sudbury, Suffolk CO10 9RW, UK | POINT Z (0.7982752 52.1071416 0) 79 | Water Street, Stamford, Lincolnshire PE9 2NJ, UK | POINT Z (-0.4752917 52.649958 0) 80 | Water Street, Cambridge, Cambridgeshire CB4 1PA, UK | POINT Z (0.1474321 52.2184911 0) 81 | Water Street, Hampstead Norreys, Thatcham, West Berkshire RG18 0RU, UK | POINT Z (-1.2408465 51.4854726 0) 82 | Water Street, Burntwood, Staffordshire WS7 1AW, UK | POINT Z (-1.9355616 52.6839693 0) 83 | Water Street, Kettering, Northamptonshire NN16, UK | POINT Z (-0.7173979 52.4008413 0) 84 | Water Street, Birmingham, West Midlands B3 1HP, UK | POINT Z (-1.9028035 52.4854385 0) 85 | Water Street, London WC2R 3LA, UK | POINT Z (-0.1136366 51.5118691 0) 86 | -------------------------------------------------------------------------------- /geofdw/__init__.py: -------------------------------------------------------------------------------- 1 | from multicorn import ForeignDataWrapper, Qual 2 | from multicorn.utils import * 3 | from ._version import __version__ 4 | from . import fdw 5 | -------------------------------------------------------------------------------- /geofdw/_version.py: -------------------------------------------------------------------------------- 1 | __version__ = "0.1.0" 2 | -------------------------------------------------------------------------------- /geofdw/base.py: -------------------------------------------------------------------------------- 1 | from multicorn import ForeignDataWrapper, Qual 2 | from multicorn.utils import log_to_postgres 3 | from logging import ERROR, INFO, DEBUG, WARNING, CRITICAL 4 | from geofdw.exception import MissingColumnError, MissingOptionError, OptionTypeError 5 | 6 | 7 | class GeoFDW(ForeignDataWrapper): 8 | def __init__(self, options, columns, srid=None): 9 | super(GeoFDW, self).__init__(options, columns) 10 | self.options = options 11 | self.columns = columns 12 | self.srid = srid 13 | 14 | def check_columns(self, columns): 15 | for column in columns: 16 | if column not in self.columns: 17 | raise MissingColumnError(column) 18 | 19 | def get_option(self, option, required=True, default=None, option_type=str): 20 | if required and option not in self.options: 21 | raise MissingOptionError(option) 22 | value = self.options.get(option, default) 23 | if value is None: 24 | return None 25 | try: 26 | return option_type(value) 27 | except ValueError as e: 28 | raise OptionTypeError(option, option_type) 29 | 30 | def get_request_options(self): 31 | if "verify" in self.options: 32 | self.verify = self.options.get("verify").lower() in ["1", "t", "true"] 33 | else: 34 | self.verify = True 35 | 36 | if "user" in self.options and "pass" in self.options: 37 | self.auth = (self.options.get("user"), self.options.get("pass")) 38 | else: 39 | self.auth = None 40 | 41 | def log(self, message, level=WARNING): 42 | log_to_postgres(message, level) 43 | -------------------------------------------------------------------------------- /geofdw/exception.py: -------------------------------------------------------------------------------- 1 | from multicorn.utils import log_to_postgres 2 | from logging import ERROR 3 | 4 | 5 | class GeoFDWError(Exception): 6 | def __init__(self, message): 7 | log_to_postgres(message, ERROR) 8 | 9 | 10 | class MissingColumnError(GeoFDWError): 11 | """ 12 | Required column missing from either __init__ or execute (e.g. GeoJSON FDW 13 | requires a geom column). 14 | """ 15 | def __init__(self, column): 16 | message = "Missing column '%s'" % column 17 | super(MissingColumnError, self).__init__(message) 18 | 19 | 20 | class MissingOptionError(GeoFDWError): 21 | """ 22 | Required option missing from __init__ (e.g. GeoJSON FDW requires a url 23 | option). 24 | """ 25 | def __init__(self, option): 26 | message = "Missing option '%s'" % option 27 | super(MissingOptionError, self).__init__(message) 28 | 29 | 30 | class OptionTypeError(GeoFDWError): 31 | """ 32 | Option has wrong type (e.g. SRID must be an integer). 33 | """ 34 | def __init__(self, option, option_type): 35 | message = "Option %s is not of type %s" % (option, option_type) 36 | super(OptionTypeError, self).__init__(message) 37 | 38 | 39 | class OptionValueError(GeoFDWError): 40 | """ 41 | Option has an invalid value. 42 | """ 43 | def __init__(self, message): 44 | super(OptionValueError, self).__init__(message) 45 | 46 | 47 | class CRSError(GeoFDWError): 48 | """ 49 | Invalid CRS. 50 | """ 51 | def __init__(self, crs): 52 | message = "Bad CRS value of %s" % crs 53 | super(CRSError, self).__init__(message) 54 | 55 | 56 | class InvalidGeometryError(GeoFDWError): 57 | pass 58 | 59 | 60 | class ValueBoundsError(GeoFDWError): 61 | pass 62 | 63 | 64 | class QueryPredicateError(GeoFDWError): 65 | pass 66 | 67 | 68 | class MissingQueryPredicateError(GeoFDWError): 69 | """ 70 | Required query predicate missing from execute (e.g. FGeocode FDW requires a 71 | predicate named query). 72 | """ 73 | def __init__(self, message): 74 | super(MissingQueryPredicateError, self).__init__(message) 75 | -------------------------------------------------------------------------------- /geofdw/fdw/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bosth/geofdw/e5e59a0da0650cc8161e6097a990bb7a0b9401d5/geofdw/fdw/__init__.py -------------------------------------------------------------------------------- /geofdw/fdw/geocode.py: -------------------------------------------------------------------------------- 1 | """ 2 | :class:`FGeocode` and `RGeocode` are foreign data wrappers for the geopy 3 | geocoding module. 4 | """ 5 | 6 | from geofdw.base import * 7 | import geopy 8 | from plpygis import Geometry, Point 9 | 10 | 11 | class _Geocode(GeoFDW): 12 | def __init__(self, options, columns): 13 | super(_Geocode, self).__init__(options, columns, srid=4326) 14 | self.service = options.get("service", "googlev3") 15 | geocoder = geopy.get_geocoder_for_service(self.service) 16 | if geocoder == geopy.geocoders.googlev3.GoogleV3: 17 | api_key = options.get("api_key") 18 | self.geocoder = geocoder(api_key=api_key) 19 | elif geocoder == geopy.geocoders.arcgis.ArcGIS: 20 | username = options.get("username") 21 | password = options.get("password") 22 | self.geocoder = geocoder(username=username, password=password) 23 | else: 24 | self.geocoder = geocoder() 25 | 26 | def get_path_keys(self): 27 | """ 28 | Query planner helper. 29 | """ 30 | return [("rank", 1), ("geom", 1), ("address", 1)] 31 | 32 | 33 | class FGeocode(_Geocode): 34 | """ 35 | The FGeocode foreign data wrapper can do forward geocoding using a number 36 | of online services. The following columns may exist in the table: query 37 | TEXT, rank INTEGER, geom GEOMETRY(POINTZ, 4326), address TEXT. 38 | 39 | Note that the geometry will be a 3d point with SRID 4326. At present, no 40 | supported geocoder returns a useful elevation (the GoogleV3 geocoder, for 41 | example, returns a static elevation of 0). 42 | """ 43 | 44 | def __init__(self, options, columns): 45 | """ 46 | Create the table that uses GoogleV3 by default or one of the following 47 | named geocoders: ArcGIS; GoogleV3; Nominatim. 48 | 49 | :param dict options: Options passed to the table creation. 50 | service: 'arcgis', 'googlev3', 'nominatim' 51 | api_key: API key for GoogleV3 (optional) 52 | username: user name for ArcGIS (optional) 53 | password: password for ArcGIS (optional) 54 | 55 | :param list columns: Columns the user has specified in PostGIS. 56 | """ 57 | super(FGeocode, self).__init__(options, columns) 58 | 59 | def execute(self, quals, columns): 60 | """ 61 | Execute the query on the geocoder. 62 | 63 | :param list quals: List of predicates from the WHERE clause of the SQL 64 | statement. The geocoder expects that one of these predicates will be of 65 | the form "query = 'Helsinki, Finland". Optionally, a bounding polygon 66 | can be used to influence the geocoder if it is supported; the following 67 | formats are recognised (and treated equivalently): 68 | 69 | geom && ST_GeomFromText('POLYGON(...)') 70 | ST_GeomFromText('POLYGON(...)') && geom 71 | geom @ ST_GeomFromText('POLYGON(...)') 72 | ST_GeomFromText('POLYGON(...)') ~ geom 73 | 74 | Other predicates may be added, but they will be evaluated in 75 | PostgreSQL and not here. 76 | 77 | :param list columns: List of columns requested in the SELECT statement. 78 | """ 79 | query, bounds = self._get_predicates(quals) 80 | 81 | if query: 82 | return self._execute(columns, query, bounds) 83 | else: 84 | return [] 85 | 86 | def _execute(self, columns, query, bounds=None): 87 | rank = 0 88 | col_geom = "geom" in columns 89 | col_addr = "address" in columns 90 | col_query = "query" in columns 91 | locations = self._get_locations(query, bounds) 92 | 93 | if locations: 94 | for location in locations: 95 | rank = rank + 1 96 | row = {"rank": rank} 97 | if col_geom: 98 | geom = Point((location.longitude, location.latitude, 99 | location.altitude), srid=self.srid) 100 | row["geom"] = geom 101 | if col_addr: 102 | row["address"] = location.address 103 | if col_query: 104 | row["query"] = query 105 | yield row 106 | 107 | def _get_predicates(self, quals): 108 | query = None 109 | bounds = None 110 | for qual in quals: 111 | if qual.field_name == "query" and qual.operator == "=": 112 | query = qual.value 113 | 114 | # note A ~ B is transformed into B @ A 115 | if qual.field_name == "geom" and qual.operator in ["&&", "@"]: 116 | bounds = Geometry(qual.value).bounds 117 | elif qual.value == "geom" and qual.operator == "&&": 118 | bounds = Geometry(qual.field_name).bounds 119 | 120 | return query, bounds 121 | 122 | def _get_locations(self, query, bounds): 123 | log_to_postgres("Geocode (%s): running query '%s' with bounds = %s" % 124 | (self.service, query, str(bounds)), DEBUG) 125 | if bounds and self.service == "googlev3": 126 | return self.geocoder.geocode(query, False, bounds=(bounds[1], bounds[0], bounds[3], bounds[2])) 127 | else: 128 | return self.geocoder.geocode(query, False) 129 | 130 | 131 | class RGeocode(_Geocode): 132 | """ 133 | The RGeocode foreign data wrapper can do reverse geocoding using a number 134 | of online services. The following columns may exist in the table: query 135 | GEOMETRY(POINT, 4326), rank INTEGER, geom GEOMETRY(POINTZ, 4326), address 136 | TEXT. 137 | 138 | Note that the geometry will be a 3d point with SRID 4326. At present, no 139 | supported geocoder returns a useful elevation (the GoogleV3 geocoder, for 140 | example, returns a static elevation of 0). 141 | """ 142 | 143 | def __init__(self, options, columns): 144 | """ 145 | Create the table that uses GoogleV3 by default or one of the following 146 | named geocoders: ArcGIS; GoogleV3; Nominatim. 147 | 148 | :param dict options: Options passed to the table creation. 149 | service: 'arcgis', 'googlev3', 'nominatim' 150 | api_key: API key for GoogleV3 (optional) 151 | username: user name for ArcGIS (optional) 152 | password: password for ArcGIS (optional) 153 | 154 | :param list columns: Columns the user has specified in PostGIS. 155 | """ 156 | super(RGeocode, self).__init__(options, columns) 157 | 158 | def execute(self, quals, columns): 159 | """ 160 | Execute the query on the geocoder. 161 | 162 | :param list quals: List of predicates from the WHERE clause of the SQL 163 | statement. The geocoder expects that one of these predicates will be of 164 | the form "query = ST_MakePoint(0, 52)" 165 | 166 | Other predicates may be added, but they will be evaluated in 167 | PostgreSQL and not here. 168 | 169 | :param list columns: List of columns requested in the SELECT statement. 170 | """ 171 | 172 | query = self._get_predicates(quals) 173 | if query: 174 | return self._execute(columns, query) 175 | else: 176 | return [] 177 | 178 | def _execute(self, columns, query): 179 | rank = 0 180 | col_geom = "geom" in columns 181 | col_addr = "address" in columns 182 | col_query = "query" in columns 183 | locations = self._get_locations(query) 184 | 185 | for location in locations: 186 | rank = rank + 1 187 | row = {"rank": rank} 188 | if col_geom: 189 | geom = Point((location.longitude, location.latitude, 190 | location.altitude), srid=self.srid) 191 | row["geom"] = geom 192 | if col_addr: 193 | row["address"] = location.address 194 | if col_query: 195 | row["query"] = query.wkb 196 | yield row 197 | 198 | def _get_predicates(self, quals): 199 | for qual in quals: 200 | if qual.field_name == "query" and qual.operator == "=": 201 | return Geometry(qual.value) 202 | return None 203 | 204 | def _get_locations(self, query): 205 | log_to_postgres("GeocodeR (%s): running query '%s'" % (self.service, 206 | query), DEBUG) 207 | return self.geocoder.reverse([query.x, query.y]) 208 | -------------------------------------------------------------------------------- /geofdw/fdw/geojson.py: -------------------------------------------------------------------------------- 1 | """ 2 | :class:`GeoJSON` is a GeoJSON foreign data wrapper. 3 | """ 4 | 5 | from geofdw.base import GeoFDW 6 | from geofdw.exception import MissingColumnError, MissingOptionError, OptionTypeError 7 | from plpygis import Geometry 8 | import json 9 | import requests 10 | 11 | 12 | class GeoJSON(GeoFDW): 13 | """ 14 | The GeoJSON foreign data wrapper can read the contents of an online GeoJSON 15 | file. The following column will exist in the table: geom GEOMETRY. 16 | Additional columns may be specified. 17 | 18 | Since column names in PostgreSQL are usually lowercase, so the wrapper will 19 | attempt case-insensitive matching between the column names and the 20 | attribute names in the GeoJSON file. 21 | 22 | Note that the geometry will use the SRID 4326 as specified by the GeoJSON 23 | standard. If you think you know better the SRID can be overridden with a 24 | table option (remember, you are *casting* the values to a new CRS, not 25 | transforming them). The GeoJSON foreign data wrapper will *not* attempt to 26 | parse any CRS defined in the GeoJSON file itself. 27 | """ 28 | def __init__(self, options, columns): 29 | """ 30 | Create the table definition based on the provided column names and 31 | options. 32 | 33 | :param dict options: Options passed to the table creation. 34 | url: location of the GeoJSON file (required) 35 | srid: custom SRID that overrides the 4326 default 36 | verify: set to false to ignore invalid SSL certificates 37 | user: user name for authentication 38 | pass: password for authentication 39 | 40 | :param list columns: Columns the user has specified in PostGIS. 41 | geom (required) 42 | """ 43 | super(GeoJSON, self).__init__(options, columns) 44 | self.check_columns(["geom"]) 45 | self.url = self.get_option("url") 46 | self.srid = self.get_option("srid", required=False, default=4326, 47 | option_type=int) 48 | self.get_request_options() 49 | 50 | def execute(self, quals, columns): 51 | """ 52 | Execute the query by reading the GeoJSON file and returning the 53 | contents based on the selected columns. 54 | 55 | :param list quals: List of predicates from the WHERE clause of the SQL 56 | statement. All filtering will happen in PostgreSQL rather than in the 57 | foreign data wrapper. 58 | 59 | :param list columns: List of columns requested in the SELECT statement. 60 | """ 61 | try: 62 | response = requests.get(self.url, auth=self.auth, 63 | verify=self.verify) 64 | except requests.exceptions.ConnectionError as e: 65 | self.log("GeoJSON FDW: unable to connect to %s" % self.url) 66 | return [] 67 | except requests.exceptions.Timeout as e: #pragma: no cover 68 | self.log("GeoJSON FDW: timeout connecting to %s" % self.url) 69 | return [] 70 | 71 | try: 72 | data = response.json() 73 | except ValueError as e: 74 | self.log("GeoJSON FDW: invalid JSON") 75 | return [] 76 | try: 77 | features = data["features"] 78 | except KeyError as e: 79 | self.log("GeoJSON FDW: invalid GeoJSON") 80 | return [] 81 | return self._execute(features, columns) 82 | 83 | def _execute(self, features, columns): 84 | if "geom" in columns: 85 | columns.remove("geom") 86 | use_geom = True 87 | else: 88 | use_geom = False 89 | for feat in features: 90 | row = {} 91 | if use_geom: 92 | gj = feat["geometry"] 93 | geom = Geometry.from_geojson(gj, srid=self.srid) 94 | row["geom"] = geom.wkb 95 | 96 | properties = feat["properties"] 97 | for p in properties.keys(): 98 | for col in columns: 99 | if col == p or col == p.lower(): 100 | row[col] = properties.get(p) 101 | break 102 | yield row 103 | -------------------------------------------------------------------------------- /geofdw/fdw/opensky.py: -------------------------------------------------------------------------------- 1 | """ 2 | :class:`OpenSky` is a foreign data wrapper for the OpenSky website. 3 | """ 4 | 5 | from geofdw.base import * 6 | import geopy 7 | from plpygis import Geometry, Point, LineString 8 | import os 9 | from datetime import datetime, timezone 10 | from dateutil import parser 11 | from requests import Session 12 | from requests.auth import HTTPBasicAuth 13 | from requests.exceptions import JSONDecodeError 14 | OSURL = "https://opensky-network.org/api" 15 | 16 | CATEGORY = { 17 | 0 : "No information at all", 18 | 1 : "No ADS-B Emitter Category Information", 19 | 2 : "Light (< 15500 lbs)", 20 | 3 : "Small (15500 to 75000 lbs)", 21 | 4 : "Large (75000 to 300000 lbs)", 22 | 5 : "High Vortex Large (aircraft such as B-757)", 23 | 6 : "Heavy (> 300000 lbs)", 24 | 7 : "High Performance (> 5g acceleration and 400 kts)", 25 | 8 : "Rotorcraft", 26 | 9 : "Glider / sailplane", 27 | 10 : "Lighter-than-air", 28 | 11 : "Parachutist / Skydiver", 29 | 12 : "Ultralight / hang-glider / paraglider", 30 | 13 : "Reserved", 31 | 14 : "Unmanned Aerial Vehicle", 32 | 15 : "Space / Trans-atmospheric vehicle", 33 | 16 : "Surface Vehicle – Emergency Vehicle", 34 | 17 : "Surface Vehicle – Service Vehicle", 35 | 18 : "Point Obstacle (includes tethered balloons)", 36 | 19 : "Cluster Obstacle", 37 | 20 : "Line Obstacle" 38 | } 39 | 40 | class _OpenSky(GeoFDW): 41 | def __init__(self, options, columns): 42 | super(_OpenSky, self).__init__(options, columns, srid=4326) 43 | osuser = options.get("osuser", os.getenv("OPENSKY_USER")) 44 | ospass = options.get("ospass", os.getenv("OPENSKY_PASS")) 45 | self.opensky = Session() 46 | if osuser and ospass: 47 | self.opensky.auth = HTTPBasicAuth(osuser, ospass) 48 | 49 | 50 | class StateVector(_OpenSky): 51 | """ 52 | """ 53 | 54 | def __init__(self, options, columns): 55 | """ 56 | Create a table containing aircraft and their current state. 57 | 58 | :param dict options: Options passed to the table creation. 59 | osuser: OpenSky user name 60 | ospass: OpenSky password 61 | 62 | :param list columns: Columns the user has specified in PostGIS. 63 | geom [POINTZ]: position of the airplane 64 | icao24 [TEXT]: the transponder address 65 | time [TIMESTAMP] 66 | callsign [TEXT] 67 | origin_country [TEXT] 68 | baro_altitude [FLOAT]: barometric altitude 69 | velocity [FLOAT]: measured in m/s 70 | true_track [FLOAT]: degrees clockwise from north 71 | vertical_rate [FLOAT]: measured in m/s 72 | squakw [TEXT]: transponder code 73 | spi [BOOLEAN]: whether flight status has a special purpose indicator 74 | on_ground [BOOLEAN] 75 | position_source [INTEGER]: 0=ADS-B, 1=ASTERIX, 2=MLAT, 3=FLARM 76 | category [INTEGER]: aircraft category 77 | category_text [TEXT]: aircraft category (description) 78 | """ 79 | super(StateVector, self).__init__(options, columns) 80 | 81 | def execute(self, quals, columns): 82 | """ 83 | Execute the query. 84 | 85 | :param list quals: List of predicates from the WHERE clause of the SQL 86 | statement. The API accepts a time (TIMESTAMP) or a icao24 (TEXT) identifier. A 87 | bounding polygon can also be used to influence the geocoder if it is 88 | supported; the following formats are recognised (and treated 89 | equivalently): 90 | 91 | geom && ST_GeomFromText('POLYGON(...)') 92 | ST_GeomFromText('POLYGON(...)') && geom 93 | geom @ ST_GeomFromText('POLYGON(...)') 94 | ST_GeomFromText('POLYGON(...)') ~ geom 95 | 96 | Other predicates may be added, but they will be evaluated in 97 | PostgreSQL and not server-side. 98 | 99 | :param list columns: List of columns requested in the SELECT statement. 100 | """ 101 | time, epoch, icao24, bounds = self._get_predicates(quals) 102 | return self._execute(columns, time, epoch, icao24, bounds) 103 | 104 | def _get_predicates(self, quals): 105 | time = None 106 | epoch = None 107 | icao24 = None 108 | bounds = None 109 | log_to_postgres("QUAL {}".format(quals), INFO) 110 | for qual in quals: 111 | if qual.field_name == "time" and qual.operator == "=": 112 | time = qual.value.replace(tzinfo=timezone.utc) 113 | epoch = int(qual.value.replace(tzinfo=timezone.utc).timestamp()) 114 | if qual.field_name == "icao24": 115 | if qual.operator == "=": 116 | icao24 = qual.value 117 | elif qual.operator == ("=", True): 118 | icao24 = qual.value 119 | 120 | # note A ~ B is transformed into B @ A 121 | if qual.field_name == "geom" and qual.operator in ["&&", "@"]: 122 | bounds = Geometry(qual.value).bounds 123 | elif qual.value == "geom" and qual.operator == "&&": 124 | bounds = Geometry(qual.field_name).bounds 125 | return time, epoch, icao24, bounds 126 | 127 | def _execute(self, columns, time, epoch, icao24, bounds=None): 128 | if "category" in columns: 129 | category = True 130 | else: 131 | category = False 132 | 133 | row = {} 134 | states = self._get_states(epoch, icao24, bounds, category=category) 135 | if not states: return [] 136 | 137 | for state in states: 138 | row["icao24"] = state[0] 139 | if time: 140 | row["time"] = time.isoformat() 141 | else: 142 | row["time"] = datetime.utcfromtimestamp(state[4]) 143 | if state[5] is not None and state[6] is not None: 144 | if state[7] is None: 145 | geom = Point((state[5], state[6]), srid=self.srid) 146 | else: 147 | geom = Point((state[5], state[6], state[13]), srid=self.srid) 148 | row["geom"] = geom 149 | row["callsign"] = state[1] 150 | row["origin_country"] = state[2] 151 | row["on_ground"] = state[8] 152 | row["velocity"] = state[9] 153 | row["true_track"] = state[10] 154 | row["vertical_rate"] = state[11] 155 | row["squawk"] = state[14] 156 | row["spi"] = state[15] 157 | row["position_source"] = state[16] 158 | if category: 159 | row["category"] = state[17] 160 | row["category_text"] = CATEGORY.get(state[17], None) 161 | yield row 162 | 163 | def _get_states(self, epoch, icao24, bounds, category): 164 | params = {} 165 | if category: 166 | params["extended"] = 1 167 | if bounds: 168 | params["lomin"] = bounds[0] 169 | params["lomax"] = bounds[1] 170 | params["lamin"] = bounds[2] 171 | params["lamax"] = bounds[3] 172 | if epoch: 173 | params["time"] = epoch 174 | if icao24: 175 | params["icao24"] = icao24 176 | 177 | response = self.opensky.get(f"{OSURL}/states/all", params=params) 178 | try: 179 | json = response.json() 180 | log_to_postgres("OPENSKY {}".format(response.url), INFO) 181 | except JSONDecodeError as e: 182 | log_to_postgres("OPENSKY {}".format(response.text), ERROR) 183 | raise e 184 | return json.get("states") 185 | 186 | def get_path_keys(self): 187 | """ 188 | Query planner helper. 189 | """ 190 | return [("geom", 100), ("time", 10), ("icao24", 1)] 191 | -------------------------------------------------------------------------------- /geofdw/fdw/randompoint.py: -------------------------------------------------------------------------------- 1 | from geofdw.base import GeoFDW 2 | from geofdw.exception import OptionValueError 3 | from plpygis import Point 4 | import random 5 | 6 | class RandomPoint(GeoFDW): 7 | """ 8 | The RandomPoint foreign data wrapper creates a number of random points. 9 | """ 10 | def __init__(self, options, columns): 11 | """ 12 | Create the table that will contain the random points. There will only be a 13 | single column geom of type GEOMETRY(POINT). 14 | 15 | :param dict options: Options passed to the table creation. 16 | min_x: Minimum value for x (required) 17 | min_y: Minimum value for y (required) 18 | max_x: Maximum value for x (required) 19 | max_y: Maximum value for y (required) 20 | num: Number of points 21 | srid: SRID of the points 22 | 23 | :param list columns: 24 | geom (required) 25 | """ 26 | super(RandomPoint, self).__init__(options, columns) 27 | self.check_columns(["geom"]) 28 | self.min_x = self.get_option("min_x", option_type=float) 29 | self.min_y = self.get_option("min_y", option_type=float) 30 | self.max_x = self.get_option("max_x", option_type=float) 31 | self.max_y = self.get_option("max_y", option_type=float) 32 | self.num = self.get_option("num", required=False, default=1, option_type=int) 33 | self.srid = self.get_option("srid", required=False, option_type=int) 34 | 35 | if self.max_x <= self.min_x or self.max_y <= self.min_y: 36 | raise OptionValueError("min must be smaller than max") 37 | 38 | def execute(self, quals, columns): 39 | for i in range(self.num): 40 | x = random.uniform(self.min_x, self.max_x) 41 | y = random.uniform(self.min_y, self.max_y) 42 | point = Point((x, y), srid=self.srid) 43 | yield { "geom" : point } 44 | -------------------------------------------------------------------------------- /geofdw/utils.py: -------------------------------------------------------------------------------- 1 | from geofdw.exception import CRSError 2 | 3 | 4 | def crs_to_srid(crs): 5 | if crs is None: 6 | return None 7 | srid = crs.lower().replace("epsg:", "") 8 | try: 9 | return int(srid) 10 | except ValueError as e: 11 | raise CRSError(crs) 12 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | geofdw 4 | """ 5 | 6 | import os 7 | import sys 8 | from setuptools import setup, find_packages 9 | from setuptools.command.test import test as TestCommand 10 | from geofdw._version import __version__ 11 | 12 | def readme(): 13 | with open("README.rst") as f: 14 | return f.read() 15 | 16 | class PyTest(TestCommand): 17 | def finalize_options(self): 18 | TestCommand.finalize_options(self) 19 | self.test_args = [] 20 | self.test_suite = True 21 | 22 | def run_tests(self): 23 | import pytest 24 | errcode = pytest.main(self.test_args) 25 | sys.exit(errcode) 26 | 27 | setup( 28 | name="geofdw", 29 | version=__version__, 30 | url="http://github.com/bosth/geofdw", 31 | license="GNU GPLv3", 32 | author="Benjamin Trigona-Harany", 33 | tests_require=["pytest"], 34 | cmdclass={"test": PyTest}, 35 | author_email="bosth@alumni.sfu.ca", 36 | description="Foreign Data Wrappers for PostGIS", 37 | long_description=readme(), 38 | packages=["geofdw"], 39 | include_package_data=True, 40 | platforms="any", 41 | test_suite="test.geofdw", 42 | classifiers = [ 43 | ], 44 | install_requires = [ 45 | "multicorn>=2.4", 46 | "geopy>=1.9.1", 47 | "requests>=2.4.0", 48 | "plpygis>=0.0.3" 49 | ], 50 | extras_require = { 51 | 'testing': ['pytest'] 52 | }, 53 | keywords='gis geographical postgis fdw postgresql' 54 | ) 55 | -------------------------------------------------------------------------------- /test.sql: -------------------------------------------------------------------------------- 1 | DROP SERVER geojson CASCADE; 2 | DROP SERVER geocode CASCADE; 3 | DROP SERVER geocode_reverse CASCADE; 4 | DROP SERVER random_point CASCADE; 5 | 6 | ------Random point 7 | CREATE SERVER random_point FOREIGN DATA WRAPPER multicorn OPTIONS (wrapper 'geofdw.RandomPoint'); 8 | CREATE FOREIGN TABLE random_points(geom geometry(point)) SERVER random_point OPTIONS (min_x '-180', min_y '-90', max_x '180', max_y '90', num '10'); 9 | SELECT ST_AsText(geom) FROM random_points; 10 | 11 | --GeoJSON 12 | CREATE SERVER geojson foreign data wrapper multicorn options ( wrapper 'geofdw.fdw.GeoJSON' ); 13 | CREATE FOREIGN TABLE geojson_simple (geom geometry, name TEXT, year TEXT) SERVER geojson OPTIONS (url 'https://raw.githubusercontent.com/MaptimeSEA/geojson/master/Dara.geojson', srid '900913'); 14 | CREATE FOREIGN TABLE geojson_countries (geom geometry, name TEXT, id TEXT) SERVER geojson OPTIONS (url 'https://raw.githubusercontent.com/johan/world.geo.json/master/countries.geo.json'); 15 | SELECT *, ST_AsText(geom) FROM geojson_simple; 16 | SELECT name, ST_NumGeometries(geom) AS polygons FROM geojson_countries ORDER BY polygons DESC LIMIT 10; 17 | 18 | --Forward Geocoding 19 | CREATE SERVER geocode FOREIGN DATA WRAPPER multicorn OPTIONS ( wrapper 'geofdw.FGeocode' ); 20 | CREATE FOREIGN TABLE gc_google (query TEXT, rank INTEGER, address TEXT, geom geometry) SERVER geocode; 21 | CREATE FOREIGN TABLE gc_arcgis (query TEXT, rank INTEGER, address TEXT, geom geometry) SERVER geocode OPTIONS ( service 'arcgis'); 22 | CREATE FOREIGN TABLE gc_nominatim (query TEXT, rank INTEGER, address TEXT, geom geometry) SERVER geocode OPTIONS ( service 'nominatim'); 23 | SELECT rank, address, ST_AsText(geom) FROM gc_nominatim WHERE query = 'canada house' AND geom && ST_GeomFromEWKT('SRID=4326;POLYGON((2 50, 2 55, -2 55, -2 50, 2 50))'); 24 | SELECT rank, address, ST_AsText(geom) FROM gc_nominatim WHERE query = 'canada house' AND ST_GeomFromEWKT('SRID=4326;POLYGON((2 50, 2 55, -2 55, -2 50, 2 50))') && geom; 25 | SELECT rank, address, ST_AsText(geom) FROM gc_nominatim WHERE query = 'canada house' AND ST_GeomFromEWKT('SRID=4326;POLYGON((2 50, 2 55, -2 55, -2 50, 2 50))') ~ geom; 26 | SELECT rank, address, ST_AsText(geom) FROM gc_nominatim WHERE query = 'canada house'; 27 | SELECT rank, address, ST_AsText(geom) FROM gc_google WHERE query = 'canada house'; 28 | SELECT rank, address, ST_AsText(geom) FROM gc_arcgis WHERE query = 'canada house'; 29 | 30 | --Reverse geocoding 31 | CREATE SERVER geocode_reverse FOREIGN DATA WRAPPER multicorn OPTIONS (wrapper 'geofdw.RGeocode'); 32 | CREATE FOREIGN TABLE gc_google_reverse (query geometry, rank INTEGER, address TEXT, geom geometry) SERVER geocode_reverse; 33 | SELECT rank, address, ST_AsText(geom) FROM gc_google_reverse WHERE query = ST_SetSRID(ST_MakePoint(52, -110), 4326); 34 | -------------------------------------------------------------------------------- /test/__init__.py: -------------------------------------------------------------------------------- 1 | from .fdw import * 2 | -------------------------------------------------------------------------------- /test/fdw/__init__.py: -------------------------------------------------------------------------------- 1 | from .randompoint import RandomPointTestCase 2 | from .geojson import GeoJSONTestCase 3 | #from .wcs import WCSTestCase 4 | -------------------------------------------------------------------------------- /test/fdw/geojson.py: -------------------------------------------------------------------------------- 1 | """ 2 | Test GeoJSON fdw 3 | """ 4 | 5 | import unittest 6 | from plpygis import Geometry, Point 7 | from geofdw.fdw import GeoJSON 8 | from geofdw.exception import MissingColumnError, MissingOptionError, OptionTypeError 9 | 10 | class GeoJSONTestCase(unittest.TestCase): 11 | 12 | EXAMPLE = 'https://raw.githubusercontent.com/colemanm/hurricanes/master/fl_2004_hurricanes.geojson' 13 | def test_missing_url(self): 14 | """ 15 | fdw.GeoJSON.__init__ missing url 16 | """ 17 | options = {} 18 | columns = ['geom'] 19 | self.assertRaises(MissingOptionError, GeoJSON, options, columns) 20 | 21 | def test_missing_geom_column(self): 22 | """ 23 | fdw.GeoJSON.__init__ missing geometry column 24 | """ 25 | options = {'url' : self.EXAMPLE} 26 | columns = [] 27 | self.assertRaises(MissingColumnError, GeoJSON, options, columns) 28 | 29 | def test_srid(self): 30 | """ 31 | fdw.GeoJSON.__init__ set custom SRID 32 | """ 33 | options = {'url' : self.EXAMPLE, 'srid' : '900913'} 34 | columns = ['geom'] 35 | fdw = GeoJSON(options, columns) 36 | self.assertEquals(fdw.srid, 900913) 37 | 38 | def test_invalid_srid(self): 39 | """ 40 | fdw.GeoJSON.__init__ set invalid SRID 41 | """ 42 | options = {'url' : self.EXAMPLE, 'srid' : 'EPSG:900913'} 43 | columns = ['geom'] 44 | self.assertRaises(OptionTypeError, GeoJSON, options, columns) 45 | 46 | def test_verify_ssl(self): 47 | """ 48 | fdw.GeoJSON.__init__ enable SSL verify 49 | """ 50 | options = {'url' : self.EXAMPLE, 'verify' : 'true'} 51 | columns = ['geom'] 52 | fdw = GeoJSON(options, columns) 53 | self.assertEquals(fdw.verify, True) 54 | 55 | def test_no_verify_ssl(self): 56 | """ 57 | fdw.GeoJSON.__init__ disable SSL verify 58 | """ 59 | options = {'url' : self.EXAMPLE, 'verify' : 'false'} 60 | columns = ['geom'] 61 | fdw = GeoJSON(options, columns) 62 | self.assertEquals(fdw.verify, False) 63 | 64 | def test_authentication(self): 65 | """ 66 | fdw.GeoJSON.__init__ use authentication 67 | """ 68 | options = {'url' : self.EXAMPLE, 'user' : 'name', 'pass' : 'secret'} 69 | columns = ['geom'] 70 | fdw = GeoJSON(options, columns) 71 | self.assertEquals(fdw.auth, ('name', 'secret')) 72 | 73 | def test_no_authentication(self): 74 | """ 75 | fdw.GeoJSON.__init__ disable authentication 76 | """ 77 | options = {'url' : self.EXAMPLE} 78 | columns = ['geom'] 79 | fdw = GeoJSON(options, columns) 80 | self.assertEquals(fdw.auth, None) 81 | 82 | def test_execute_bad_url(self): 83 | """ 84 | fdw.GeoJSON.execute non-existant URL 85 | """ 86 | options = {'url' : 'http://d.xyz'} 87 | columns = ['geom'] 88 | fdw = GeoJSON(options, columns) 89 | rows = fdw.execute([], columns) 90 | self.assertListEqual(rows, []) 91 | 92 | def test_not_json(self): 93 | """ 94 | fdw.GeoJSON.execute receive non-JSON response 95 | """ 96 | options = {'url' : 'http://raw.githubusercontent.com'} 97 | columns = ['geom'] 98 | fdw = GeoJSON(options, columns) 99 | rows = fdw.execute([], columns) 100 | self.assertListEqual(rows, []) 101 | 102 | def test_not_geojson(self): 103 | """ 104 | fdw.GeoJSON.execute receive non-GeoJSON response 105 | """ 106 | options = {'url' : 'https://raw.githubusercontent.com/fge/sample-json-schemas/master/json-home/json-home.json'} 107 | columns = ['geom'] 108 | fdw = GeoJSON(options, columns) 109 | rows = fdw.execute([], columns) 110 | self.assertListEqual(rows, []) 111 | 112 | def test_geojson(self): 113 | """ 114 | fdw.GeoJSON.execute receive GeoJSON response 115 | """ 116 | options = {'url' : self.EXAMPLE} 117 | columns = ['geom'] 118 | fdw = GeoJSON(options, columns) 119 | rows = fdw.execute([], columns) 120 | for row in rows: 121 | wkb = row["geom"] 122 | geom = Geometry(wkb) 123 | self.assertIsInstance(geom, Point) 124 | 125 | def test_geojson_attribute(self): 126 | """ 127 | fdw.GeoJSON.execute receive GeoJSON response with non-spatial attribute 128 | """ 129 | options = {'url' : self.EXAMPLE} 130 | columns = ['geom', 'NAME'] 131 | fdw = GeoJSON(options, columns) 132 | rows = fdw.execute([], ['NAME']) 133 | for row in rows: 134 | self.assertIn(row['NAME'], ['Alex', 'Bonnie', 'Charley', 'Danielle', 'Earl', 'Frances', 'Gaston', 'Hermine', 'Ivan', 'Tropical Depression 2', 'Tropical Depression 10', 'Jeanne', 'Karl', 'Lisa', 'Matthew', 'Nicole', 'Otto']) 135 | -------------------------------------------------------------------------------- /test/fdw/randompoint.py: -------------------------------------------------------------------------------- 1 | """ 2 | Test randompoint fdw 3 | """ 4 | 5 | import unittest 6 | 7 | from plpygis import Geometry 8 | from geofdw.fdw import RandomPoint 9 | from geofdw.exception import MissingOptionError, OptionTypeError, OptionValueError 10 | 11 | class RandomPointTestCase(unittest.TestCase): 12 | def test_missing_option(self): 13 | """ 14 | fdw.RandomPoint.__init__ missing option 15 | """ 16 | options = {} 17 | columns = ['geom'] 18 | self.assertRaises(MissingOptionError, RandomPoint, options, columns) 19 | 20 | def test_bad_option_type(self): 21 | """ 22 | fdw.RandomPoint.__init__ incorrect option type 23 | """ 24 | options = {'min_x':'test'} 25 | columns = ['geom'] 26 | self.assertRaises(OptionTypeError, RandomPoint, options, columns) 27 | 28 | def test_bad_option_value(self): 29 | """ 30 | fdw.RandomPoint.__init__ incorrect option value 31 | """ 32 | options = {'min_x':1, 'max_x':0, 'min_y':0, 'max_y':1} 33 | columns = ['geom'] 34 | self.assertRaises(OptionValueError, RandomPoint, options, columns) 35 | 36 | def test_options(self): 37 | """ 38 | fdw.RandomPoint.__init__ options 39 | """ 40 | options = {'min_x':10, 'max_x':20, 'min_y':30, 'max_y':40, 'num': 99, 'srid':4326} 41 | columns = ['geom'] 42 | fdw = RandomPoint(options, columns) 43 | self.assertEquals(fdw.min_x, 10) 44 | self.assertEquals(fdw.max_x, 20) 45 | self.assertEquals(fdw.min_y, 30) 46 | self.assertEquals(fdw.max_y, 40) 47 | self.assertEquals(fdw.num, 99) 48 | self.assertEquals(fdw.srid, 4326) 49 | 50 | def test_execute(self): 51 | """ 52 | fdw.RandomPoint.execute check results 53 | """ 54 | options = {'min_x':10, 'max_x':20, 'min_y':30, 'max_y':40, 'num': 99, 'srid':4326} 55 | columns = ['geom'] 56 | fdw = RandomPoint(options, columns) 57 | rows = fdw.execute([], columns) 58 | for row in rows: 59 | wkb = str(row["geom"]) 60 | point = Geometry(wkb) 61 | self.assertTrue(10 <= point.x <= 20) 62 | self.assertTrue(30 <= point.y <= 40) 63 | -------------------------------------------------------------------------------- /test/test_geofdw.py: -------------------------------------------------------------------------------- 1 | """ 2 | Test basic geofdw 3 | """ 4 | 5 | import unittest 6 | 7 | from geofdw.base import GeoFDW 8 | 9 | class GeoFDWTestCase(unittest.TestCase): 10 | def test_init(self): 11 | """ 12 | GeoFDW 13 | """ 14 | fdw = GeoFDW({}, ['geom']) 15 | self.assertListEqual(fdw.columns, ['geom']) 16 | -------------------------------------------------------------------------------- /test/test_utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | Test geofdw utils 3 | """ 4 | 5 | import unittest 6 | from geofdw.utils import * 7 | 8 | class utils(unittest.TestCase): 9 | def test_crs_to_srid_none(self): 10 | """ 11 | crs_to_srid converting missing CRS to a SRID 12 | """ 13 | srid = crs_to_srid(None) 14 | self.assertEquals(srid, None) 15 | 16 | def test_crs_to_srid_lower(self): 17 | """ 18 | crs_to_srid converting lower-case CRS to a SRID 19 | """ 20 | srid = crs_to_srid('epsg:4326') 21 | self.assertEquals(srid, 4326) 22 | 23 | def test_crs_to_srid_upper(self): 24 | """ 25 | crs_to_srid converting upper-case CRS to a SRID 26 | """ 27 | srid = crs_to_srid('EPSG:4326') 28 | self.assertEquals(srid, 4326) 29 | --------------------------------------------------------------------------------