├── .gitignore ├── beacondump ├── __init__.py ├── dump.py └── test.py └── requirements.txt /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | -------------------------------------------------------------------------------- /beacondump/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | bs4 == 0.0.1 2 | -------------------------------------------------------------------------------- /beacondump/dump.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import sys 3 | import json 4 | import copy 5 | import re 6 | import collections 7 | from argparse import ArgumentParser 8 | import http.client 9 | import urllib.parse 10 | import bs4 11 | from geomet import wkt as parseWkt 12 | 13 | 14 | BEACON_HEADERS = { 15 | 'Content-Type': 'application/json', 16 | 'User-Agent': 'OA', 17 | } 18 | 19 | BODY_TEMPLATE = { 20 | "layerId": None, 21 | "useSelection": False, 22 | "ext": { 23 | "minx": 0, 24 | "miny": 0, 25 | "maxx": 40000000, 26 | "maxy": 40000000 27 | }, 28 | "wkt": None, 29 | "spatialRelation": 1, 30 | "featureLimit": 1 31 | } 32 | 33 | name_value_pattern = re.compile(r'^(\w+) = (.*)$', re.M) 34 | coordinate_pattern = re.compile(r'(?P-?\d+(\.\d+)?)\s+(?P-?\d+(\.\d+)?)') 35 | 36 | 37 | def get_parameters(url): 38 | scheme, host, path, _, query, _ = urllib.parse.urlparse(url) 39 | layer_path = urllib.parse.urlunparse(('', '', path, None, query, None)) 40 | 41 | if scheme == 'https': 42 | conn, layer_path = http.client.HTTPSConnection(host), layer_path 43 | 44 | if scheme == 'http': 45 | conn, layer_path = http.client.HTTPConnection(host), layer_path 46 | 47 | conn.request( 48 | 'GET', url=layer_path, 49 | headers=BEACON_HEADERS 50 | ) 51 | 52 | resp = conn.getresponse() 53 | 54 | if resp.status not in range(200, 299): 55 | raise RuntimeError('Bad status in %s' % url) 56 | 57 | page = resp.read().decode('utf-8') 58 | match = re.search(r'var mapConfig = ([^;]+?);', page) 59 | return json.loads(match.group(1)) 60 | 61 | 62 | def get_connection(raw_url): 63 | ''' Return an HTTPConnection and URL path for a starting Beacon URL. 64 | 65 | Expects a raw URL similar to: 66 | https://beacon.schneidercorp.com/api/beaconCore/GetVectorLayer?QPS=xxxx 67 | ''' 68 | # Safari developer tools sneaks in some zero-width spaces: 69 | # http://www.fileformat.info/info/unicode/char/200B/index.htm 70 | url = raw_url.replace('\u200b', '') 71 | 72 | scheme, host, path, _, query, _ = urllib.parse.urlparse(url) 73 | layer_path = urllib.parse.urlunparse(('', '', path, None, query, None)) 74 | 75 | if scheme == 'https': 76 | return http.client.HTTPSConnection(host), layer_path 77 | 78 | if scheme == 'http': 79 | return http.client.HTTPConnection(host), layer_path 80 | 81 | 82 | def get_starting_bbox(conn, layer_path, layer_id, radius_km=200): 83 | ''' Retrieves a bounding box tuple for a Beacon layer and radius in km. 84 | 85 | This is meant to be an overly-large, generous bbox that should 86 | encompass any reasonable county or city data source. 87 | ''' 88 | body = copy.deepcopy(BODY_TEMPLATE) 89 | body['layerId'] = int(layer_id) 90 | conn.request( 91 | 'POST', url=layer_path, 92 | body=json.dumps(body), 93 | headers=BEACON_HEADERS 94 | ) 95 | 96 | resp = conn.getresponse() 97 | 98 | if resp.status not in range(200, 299): 99 | raise RuntimeError('Bad status in get_starting_bbox') 100 | 101 | results = json.load(resp) 102 | wkt = results.get('d', [{}])[0].get('WktGeometry', None) 103 | 104 | if not wkt: 105 | raise RuntimeError('Missing WktGeometry in get_starting_bbox') 106 | 107 | match = coordinate_pattern.search(wkt) 108 | 109 | if not match: 110 | raise RuntimeError('Unparseable WktGeometry in get_started_bbox') 111 | 112 | x, y = float(match.group('x')), float(match.group('y')) 113 | xmin, ymin = x - radius_km * 1000, y - radius_km * 1000 114 | xmax, ymax = x + radius_km * 1000, y + radius_km * 1000 115 | 116 | return xmin, ymin, xmax, ymax 117 | 118 | 119 | def partition_bbox(xmin, ymin, xmax, ymax): 120 | ''' Cut a bounding box into four smaller bounding boxes. 121 | ''' 122 | xmid, ymid = xmin/2 + xmax/2, ymin/2 + ymax/2 123 | 124 | return [ 125 | (xmin, ymin, xmid, ymid), 126 | (xmin, ymid, xmid, ymax), 127 | (xmid, ymin, xmax, ymid), 128 | (xmid, ymid, xmax, ymax), 129 | ] 130 | 131 | 132 | def get_features(conn, layer_path, layer_id, bbox, limit=0, depth=0): 133 | ''' Return a list of features after geographically searching a layer. 134 | ''' 135 | body = copy.deepcopy(BODY_TEMPLATE) 136 | body['layerId'], body['featureLimit'] = int(layer_id), limit 137 | body['ext'] = dict(minx=bbox[0], miny=bbox[1], maxx=bbox[2], maxy=bbox[3]) 138 | 139 | conn.request( 140 | 'POST', url=layer_path, 141 | body=json.dumps(body), 142 | headers=BEACON_HEADERS 143 | ) 144 | 145 | resp = conn.getresponse() 146 | 147 | if resp.status not in range(200, 299): 148 | raise RuntimeError('Bad status in get_features') 149 | 150 | records = json.load(resp).get('d', []) 151 | 152 | if limit == 0: 153 | # This is our first time through and we don't actually know how many 154 | # things there are. Assume that the current count is the limit. 155 | limit = len(records) 156 | 157 | if len(records) >= limit: 158 | # There are too many records, recurse! 159 | # This also happens the first time through before we know anything. 160 | bbox1, bbox2, bbox3, bbox4 = partition_bbox(*bbox) 161 | return get_features(conn, layer_path, layer_id, bbox1, limit, depth+1) \ 162 | + get_features(conn, layer_path, layer_id, bbox2, limit, depth+1) \ 163 | + get_features(conn, layer_path, layer_id, bbox3, limit, depth+1) \ 164 | + get_features(conn, layer_path, layer_id, bbox4, limit, depth+1) 165 | 166 | # We are good, make some GeoJSON. 167 | print(' ' * depth, 'found', len(records), 'in', bbox, file=sys.stderr) 168 | return [make_feature(record) for record in records] 169 | 170 | 171 | def extract_properties(record): 172 | ''' Get a dictionary of GeoJSON feature properties for a record. 173 | ''' 174 | properties = collections.OrderedDict(**record) 175 | 176 | html1 = record.get('TipHtml', '').replace('\r\n', '\n') 177 | html2 = record.get('ResultHtml', '').replace('\r\n', '\n') 178 | 179 | soup1 = bs4.BeautifulSoup(html1, 'html.parser') 180 | soup2 = bs4.BeautifulSoup(html2, 'html.parser') 181 | 182 | for text in soup1.find_all(text=name_value_pattern): 183 | properties.update({k: v for (k, v) in name_value_pattern.findall(text)}) 184 | 185 | for b in soup1('b'): 186 | properties[b.text.strip()] = b.nextSibling.strip(' -=') 187 | 188 | for text in soup2.find_all(text=name_value_pattern): 189 | properties.update({k: v for (k, v) in name_value_pattern.findall(text)}) 190 | 191 | return properties 192 | 193 | 194 | def extract_geometry(record): 195 | ''' Get a GeoJSON geometry object for a record. 196 | ''' 197 | prop = extract_properties(record) 198 | 199 | try: 200 | geom = parseWkt.loads(prop['WktGeometry']) 201 | except KeyError: 202 | geom = dict(type='Point', coordinates=[float(prop['Long']), float(prop['Lat'])]) 203 | except ValueError: 204 | geom = None 205 | 206 | return geom 207 | 208 | 209 | def make_feature(record): 210 | ''' Get a complete GeoJSON feature object for a record. 211 | ''' 212 | return dict( 213 | type='Feature', 214 | id=record.get('Key'), 215 | geometry=extract_geometry(record), 216 | properties=extract_properties(record) 217 | ) 218 | 219 | 220 | def main(): 221 | parser = ArgumentParser() 222 | parser.add_argument('url', help='map URL') 223 | parser.add_argument('file', help='output file') 224 | args = parser.parse_args() 225 | 226 | params = get_parameters(args.url) 227 | url = 'https://beacon.schneidercorp.com/api/beaconCore/GetVectorLayer?QPS=' + params['QPS'] 228 | layer_id = params['LayerId'] 229 | 230 | conn, layer_path = get_connection(url) 231 | bbox = get_starting_bbox(conn, layer_path, layer_id) 232 | print(bbox, file=sys.stderr) 233 | 234 | features = get_features(conn, layer_path, layer_id, bbox) 235 | geojson = dict(type='FeatureCollection', features=list(features)) 236 | 237 | with open(args.file, 'w') as file: 238 | json.dump(geojson, file, indent=2) 239 | 240 | 241 | if __name__ == '__main__': 242 | main() 243 | -------------------------------------------------------------------------------- /beacondump/test.py: -------------------------------------------------------------------------------- 1 | import unittest, unittest.mock, json 2 | from . import dump 3 | 4 | Records = dict( 5 | F1 = '''{"Key":"1","Fields":{},"WktGeometry":"MULTIPOINT ((1408409.02 265911.91))","TipHtml":"Address = 160 VILLA DR\\r\\n\\u003cbr\\u003e\\r\\nBLDING_SUI = \\r\\n\\u003cbr\\u003e\\r\\nCITY_LIMIT = TANEY COUNTY\\r\\n\\u003cbr\\u003e\\r\\nCOMMUNITY = HOLLISTER\\r\\n\\u003cbr\\u003e\\r\\ncreated_date = \\r\\n\\u003cbr\\u003e\\r\\ncreated_user = \\r\\n\\u003cbr\\u003e\\r\\nDATAFILE = A053003.ssf\\r\\n\\u003cbr\\u003e\\r\\nEASTING = 1408483.1000000001\\r\\n\\u003cbr\\u003e\\r\\nEmail_Dist = \\r\\n\\u003cbr\\u003e\\r\\nESN = 12_166\\r\\n\\u003cbr\\u003e\\r\\nFEAT_NAME = Address_\\r\\n\\u003cbr\\u003e\\r\\nGPS_DATE = 1054252800000\\r\\n\\u003cbr\\u003e\\r\\nGPS_HEIGHT = 0\\r\\n\\u003cbr\\u003e\\r\\nGPS_TIME = 01:55:53pm\\r\\n\\u003cbr\\u003e\\r\\nlast_edited_date = 1490699094000\\r\\n\\u003cbr\\u003e\\r\\nlast_edited_user = ISGIS\\r\\n\\u003cbr\\u003e\\r\\nLat = 36.561191497240003\\r\\n\\u003cbr\\u003e\\r\\nLong = -93.289987614449998\\r\\n\\u003cbr\\u003e\\r\\nNORTHING = 265915.70400000003\\r\\n\\u003cbr\\u003e\\r\\nNOTES = \\r\\n\\u003cbr\\u003e\\r\\nNotes_2 = \\r\\n\\u003cbr\\u003e\\r\\nPoints_ID = 1\\r\\n\\u003cbr\\u003e\\r\\nPROPERTY_N = \\r\\n\\u003cbr\\u003e\\r\\nSTATE = MO\\r\\n\\u003cbr\\u003e\\r\\nStatus = \\r\\n\\u003cbr\\u003e\\r\\nSTREET_DI2 = \\r\\n\\u003cbr\\u003e\\r\\nSTREET_DIR = \\r\\n\\u003cbr\\u003e\\r\\nSTREET_NAM = VILLA\\r\\n\\u003cbr\\u003e\\r\\nSTREET_NUM = 160\\r\\n\\u003cbr\\u003e\\r\\nSTREET_SUF = DR\\r\\n\\u003cbr\\u003e\\r\\nUNIT_OR_LO = 2\\r\\n\\u003cbr\\u003e\\r\\nZIP_CODE = 65672\\r\\n\\u003cbr\\u003e\\r\\n","ResultHtml":"Address = 160 VILLA DR\\r\\n\\u003cbr\\u003e\\r\\nBLDING_SUI = \\r\\n\\u003cbr\\u003e\\r\\nCITY_LIMIT = TANEY COUNTY\\r\\n\\u003cbr\\u003e\\r\\nCOMMUNITY = HOLLISTER\\r\\n\\u003cbr\\u003e\\r\\ncreated_date = \\r\\n\\u003cbr\\u003e\\r\\ncreated_user = \\r\\n\\u003cbr\\u003e\\r\\nDATAFILE = A053003.ssf\\r\\n\\u003cbr\\u003e\\r\\nEASTING = 1408483.1000000001\\r\\n\\u003cbr\\u003e\\r\\nEmail_Dist = \\r\\n\\u003cbr\\u003e\\r\\nESN = 12_166\\r\\n\\u003cbr\\u003e\\r\\nFEAT_NAME = Address_\\r\\n\\u003cbr\\u003e\\r\\nGPS_DATE = 1054252800000\\r\\n\\u003cbr\\u003e\\r\\nGPS_HEIGHT = 0\\r\\n\\u003cbr\\u003e\\r\\nGPS_TIME = 01:55:53pm\\r\\n\\u003cbr\\u003e\\r\\nlast_edited_date = 1490699094000\\r\\n\\u003cbr\\u003e\\r\\nlast_edited_user = ISGIS\\r\\n\\u003cbr\\u003e\\r\\nLat = 36.561191497240003\\r\\n\\u003cbr\\u003e\\r\\nLong = -93.289987614449998\\r\\n\\u003cbr\\u003e\\r\\nNORTHING = 265915.70400000003\\r\\n\\u003cbr\\u003e\\r\\nNOTES = \\r\\n\\u003cbr\\u003e\\r\\nNotes_2 = \\r\\n\\u003cbr\\u003e\\r\\nPoints_ID = 1\\r\\n\\u003cbr\\u003e\\r\\nPROPERTY_N = \\r\\n\\u003cbr\\u003e\\r\\nSTATE = MO\\r\\n\\u003cbr\\u003e\\r\\nStatus = \\r\\n\\u003cbr\\u003e\\r\\nSTREET_DI2 = \\r\\n\\u003cbr\\u003e\\r\\nSTREET_DIR = \\r\\n\\u003cbr\\u003e\\r\\nSTREET_NAM = VILLA\\r\\n\\u003cbr\\u003e\\r\\nSTREET_NUM = 160\\r\\n\\u003cbr\\u003e\\r\\nSTREET_SUF = DR\\r\\n\\u003cbr\\u003e\\r\\nUNIT_OR_LO = 2\\r\\n\\u003cbr\\u003e\\r\\nZIP_CODE = 65672\\r\\n\\u003cbr\\u003e\\r\\n","ParentId":null,"ChildIds":null}''', 6 | F2 = '''{"Key":"2","Fields":{},"WktGeometry":"MULTIPOINT ((1408481.6 265803.22))","TipHtml":"Address = 178 VILLA DR\\r\\n\\u003cbr\\u003e\\r\\nBLDING_SUI = \\r\\n\\u003cbr\\u003e\\r\\nCITY_LIMIT = TANEY COUNTY\\r\\n\\u003cbr\\u003e\\r\\nCOMMUNITY = HOLLISTER\\r\\n\\u003cbr\\u003e\\r\\ncreated_date = \\r\\n\\u003cbr\\u003e\\r\\ncreated_user = \\r\\n\\u003cbr\\u003e\\r\\nDATAFILE = A053003.ssf\\r\\n\\u003cbr\\u003e\\r\\nEASTING = 1408554.861\\r\\n\\u003cbr\\u003e\\r\\nEmail_Dist = \\r\\n\\u003cbr\\u003e\\r\\nESN = 12_166\\r\\n\\u003cbr\\u003e\\r\\nFEAT_NAME = Address_\\r\\n\\u003cbr\\u003e\\r\\nGPS_DATE = 1054252800000\\r\\n\\u003cbr\\u003e\\r\\nGPS_HEIGHT = 0\\r\\n\\u003cbr\\u003e\\r\\nGPS_TIME = 01:56:10pm\\r\\n\\u003cbr\\u003e\\r\\nlast_edited_date = 1490699094000\\r\\n\\u003cbr\\u003e\\r\\nlast_edited_user = ISGIS\\r\\n\\u003cbr\\u003e\\r\\nLat = 36.560894620340001\\r\\n\\u003cbr\\u003e\\r\\nLong = -93.289737463319995\\r\\n\\u003cbr\\u003e\\r\\nNORTHING = 265782.38500000001\\r\\n\\u003cbr\\u003e\\r\\nNOTES = \\r\\n\\u003cbr\\u003e\\r\\nNotes_2 = \\r\\n\\u003cbr\\u003e\\r\\nPoints_ID = 2\\r\\n\\u003cbr\\u003e\\r\\nPROPERTY_N = \\r\\n\\u003cbr\\u003e\\r\\nSTATE = MO\\r\\n\\u003cbr\\u003e\\r\\nStatus = \\r\\n\\u003cbr\\u003e\\r\\nSTREET_DI2 = \\r\\n\\u003cbr\\u003e\\r\\nSTREET_DIR = \\r\\n\\u003cbr\\u003e\\r\\nSTREET_NAM = VILLA\\r\\n\\u003cbr\\u003e\\r\\nSTREET_NUM = 178\\r\\n\\u003cbr\\u003e\\r\\nSTREET_SUF = DR\\r\\n\\u003cbr\\u003e\\r\\nUNIT_OR_LO = 4\\r\\n\\u003cbr\\u003e\\r\\nZIP_CODE = 65672\\r\\n\\u003cbr\\u003e\\r\\n","ResultHtml":"Address = 178 VILLA DR\\r\\n\\u003cbr\\u003e\\r\\nBLDING_SUI = \\r\\n\\u003cbr\\u003e\\r\\nCITY_LIMIT = TANEY COUNTY\\r\\n\\u003cbr\\u003e\\r\\nCOMMUNITY = HOLLISTER\\r\\n\\u003cbr\\u003e\\r\\ncreated_date = \\r\\n\\u003cbr\\u003e\\r\\ncreated_user = \\r\\n\\u003cbr\\u003e\\r\\nDATAFILE = A053003.ssf\\r\\n\\u003cbr\\u003e\\r\\nEASTING = 1408554.861\\r\\n\\u003cbr\\u003e\\r\\nEmail_Dist = \\r\\n\\u003cbr\\u003e\\r\\nESN = 12_166\\r\\n\\u003cbr\\u003e\\r\\nFEAT_NAME = Address_\\r\\n\\u003cbr\\u003e\\r\\nGPS_DATE = 1054252800000\\r\\n\\u003cbr\\u003e\\r\\nGPS_HEIGHT = 0\\r\\n\\u003cbr\\u003e\\r\\nGPS_TIME = 01:56:10pm\\r\\n\\u003cbr\\u003e\\r\\nlast_edited_date = 1490699094000\\r\\n\\u003cbr\\u003e\\r\\nlast_edited_user = ISGIS\\r\\n\\u003cbr\\u003e\\r\\nLat = 36.560894620340001\\r\\n\\u003cbr\\u003e\\r\\nLong = -93.289737463319995\\r\\n\\u003cbr\\u003e\\r\\nNORTHING = 265782.38500000001\\r\\n\\u003cbr\\u003e\\r\\nNOTES = \\r\\n\\u003cbr\\u003e\\r\\nNotes_2 = \\r\\n\\u003cbr\\u003e\\r\\nPoints_ID = 2\\r\\n\\u003cbr\\u003e\\r\\nPROPERTY_N = \\r\\n\\u003cbr\\u003e\\r\\nSTATE = MO\\r\\n\\u003cbr\\u003e\\r\\nStatus = \\r\\n\\u003cbr\\u003e\\r\\nSTREET_DI2 = \\r\\n\\u003cbr\\u003e\\r\\nSTREET_DIR = \\r\\n\\u003cbr\\u003e\\r\\nSTREET_NAM = VILLA\\r\\n\\u003cbr\\u003e\\r\\nSTREET_NUM = 178\\r\\n\\u003cbr\\u003e\\r\\nSTREET_SUF = DR\\r\\n\\u003cbr\\u003e\\r\\nUNIT_OR_LO = 4\\r\\n\\u003cbr\\u003e\\r\\nZIP_CODE = 65672\\r\\n\\u003cbr\\u003e\\r\\n","ParentId":null,"ChildIds":null}''', 7 | F3 = '''{"Key":"3","Fields":{},"WktGeometry":"MULTIPOINT ((1408605.34 265645.73))","TipHtml":"Address = 200 VILLA DR\\r\\n\\u003cbr\\u003e\\r\\nBLDING_SUI = \\r\\n\\u003cbr\\u003e\\r\\nCITY_LIMIT = TANEY COUNTY\\r\\n\\u003cbr\\u003e\\r\\nCOMMUNITY = HOLLISTER\\r\\n\\u003cbr\\u003e\\r\\ncreated_date = \\r\\n\\u003cbr\\u003e\\r\\ncreated_user = \\r\\n\\u003cbr\\u003e\\r\\nDATAFILE = A053003.ssf\\r\\n\\u003cbr\\u003e\\r\\nEASTING = 1408633.564\\r\\n\\u003cbr\\u003e\\r\\nEmail_Dist = \\r\\n\\u003cbr\\u003e\\r\\nESN = 12_166\\r\\n\\u003cbr\\u003e\\r\\nFEAT_NAME = Address_\\r\\n\\u003cbr\\u003e\\r\\nGPS_DATE = 1054252800000\\r\\n\\u003cbr\\u003e\\r\\nGPS_HEIGHT = 0\\r\\n\\u003cbr\\u003e\\r\\nGPS_TIME = 01:56:28pm\\r\\n\\u003cbr\\u003e\\r\\nlast_edited_date = 1490699094000\\r\\n\\u003cbr\\u003e\\r\\nlast_edited_user = ISGIS\\r\\n\\u003cbr\\u003e\\r\\nLat = 36.560464849970003\\r\\n\\u003cbr\\u003e\\r\\nLong = -93.289311785600006\\r\\n\\u003cbr\\u003e\\r\\nNORTHING = 265727.73100000003\\r\\n\\u003cbr\\u003e\\r\\nNOTES = \\r\\n\\u003cbr\\u003e\\r\\nNotes_2 = \\r\\n\\u003cbr\\u003e\\r\\nPoints_ID = 3\\r\\n\\u003cbr\\u003e\\r\\nPROPERTY_N = \\r\\n\\u003cbr\\u003e\\r\\nSTATE = MO\\r\\n\\u003cbr\\u003e\\r\\nStatus = \\r\\n\\u003cbr\\u003e\\r\\nSTREET_DI2 = \\r\\n\\u003cbr\\u003e\\r\\nSTREET_DIR = \\r\\n\\u003cbr\\u003e\\r\\nSTREET_NAM = VILLA\\r\\n\\u003cbr\\u003e\\r\\nSTREET_NUM = 200\\r\\n\\u003cbr\\u003e\\r\\nSTREET_SUF = DR\\r\\n\\u003cbr\\u003e\\r\\nUNIT_OR_LO = 1\\r\\n\\u003cbr\\u003e\\r\\nZIP_CODE = 65672\\r\\n\\u003cbr\\u003e\\r\\n","ResultHtml":"Address = 200 VILLA DR\\r\\n\\u003cbr\\u003e\\r\\nBLDING_SUI = \\r\\n\\u003cbr\\u003e\\r\\nCITY_LIMIT = TANEY COUNTY\\r\\n\\u003cbr\\u003e\\r\\nCOMMUNITY = HOLLISTER\\r\\n\\u003cbr\\u003e\\r\\ncreated_date = \\r\\n\\u003cbr\\u003e\\r\\ncreated_user = \\r\\n\\u003cbr\\u003e\\r\\nDATAFILE = A053003.ssf\\r\\n\\u003cbr\\u003e\\r\\nEASTING = 1408633.564\\r\\n\\u003cbr\\u003e\\r\\nEmail_Dist = \\r\\n\\u003cbr\\u003e\\r\\nESN = 12_166\\r\\n\\u003cbr\\u003e\\r\\nFEAT_NAME = Address_\\r\\n\\u003cbr\\u003e\\r\\nGPS_DATE = 1054252800000\\r\\n\\u003cbr\\u003e\\r\\nGPS_HEIGHT = 0\\r\\n\\u003cbr\\u003e\\r\\nGPS_TIME = 01:56:28pm\\r\\n\\u003cbr\\u003e\\r\\nlast_edited_date = 1490699094000\\r\\n\\u003cbr\\u003e\\r\\nlast_edited_user = ISGIS\\r\\n\\u003cbr\\u003e\\r\\nLat = 36.560464849970003\\r\\n\\u003cbr\\u003e\\r\\nLong = -93.289311785600006\\r\\n\\u003cbr\\u003e\\r\\nNORTHING = 265727.73100000003\\r\\n\\u003cbr\\u003e\\r\\nNOTES = \\r\\n\\u003cbr\\u003e\\r\\nNotes_2 = \\r\\n\\u003cbr\\u003e\\r\\nPoints_ID = 3\\r\\n\\u003cbr\\u003e\\r\\nPROPERTY_N = \\r\\n\\u003cbr\\u003e\\r\\nSTATE = MO\\r\\n\\u003cbr\\u003e\\r\\nStatus = \\r\\n\\u003cbr\\u003e\\r\\nSTREET_DI2 = \\r\\n\\u003cbr\\u003e\\r\\nSTREET_DIR = \\r\\n\\u003cbr\\u003e\\r\\nSTREET_NAM = VILLA\\r\\n\\u003cbr\\u003e\\r\\nSTREET_NUM = 200\\r\\n\\u003cbr\\u003e\\r\\nSTREET_SUF = DR\\r\\n\\u003cbr\\u003e\\r\\nUNIT_OR_LO = 1\\r\\n\\u003cbr\\u003e\\r\\nZIP_CODE = 65672\\r\\n\\u003cbr\\u003e\\r\\n","ParentId":null,"ChildIds":null}''', 8 | F4 = '''{"Key":"4","Fields":{},"WktGeometry":"MULTIPOINT ((1408616.28 265851.91))","TipHtml":"Address = 179 VILLA DR\\r\\n\\u003cbr\\u003e\\r\\nBLDING_SUI = \\r\\n\\u003cbr\\u003e\\r\\nCITY_LIMIT = TANEY COUNTY\\r\\n\\u003cbr\\u003e\\r\\nCOMMUNITY = HOLLISTER\\r\\n\\u003cbr\\u003e\\r\\ncreated_date = \\r\\n\\u003cbr\\u003e\\r\\ncreated_user = \\r\\n\\u003cbr\\u003e\\r\\nDATAFILE = A053003.ssf\\r\\n\\u003cbr\\u003e\\r\\nEASTING = 1408571.247\\r\\n\\u003cbr\\u003e\\r\\nEmail_Dist = \\r\\n\\u003cbr\\u003e\\r\\nESN = 12_166\\r\\n\\u003cbr\\u003e\\r\\nFEAT_NAME = Address_\\r\\n\\u003cbr\\u003e\\r\\nGPS_DATE = 1054252800000\\r\\n\\u003cbr\\u003e\\r\\nGPS_HEIGHT = 0\\r\\n\\u003cbr\\u003e\\r\\nGPS_TIME = 01:56:42pm\\r\\n\\u003cbr\\u003e\\r\\nlast_edited_date = 1490699094000\\r\\n\\u003cbr\\u003e\\r\\nlast_edited_user = ISGIS\\r\\n\\u003cbr\\u003e\\r\\nLat = NULL\\r\\n\\u003cbr\\u003e\\r\\nLong = NULL\\r\\n\\u003cbr\\u003e\\r\\nNORTHING = 265777.30800000002\\r\\n\\u003cbr\\u003e\\r\\nNOTES = \\r\\n\\u003cbr\\u003e\\r\\nNotes_2 = \\r\\n\\u003cbr\\u003e\\r\\nPoints_ID = 4\\r\\n\\u003cbr\\u003e\\r\\nPROPERTY_N = \\r\\n\\u003cbr\\u003e\\r\\nSTATE = MO\\r\\n\\u003cbr\\u003e\\r\\nStatus = \\r\\n\\u003cbr\\u003e\\r\\nSTREET_DI2 = \\r\\n\\u003cbr\\u003e\\r\\nSTREET_DIR = \\r\\n\\u003cbr\\u003e\\r\\nSTREET_NAM = VILLA\\r\\n\\u003cbr\\u003e\\r\\nSTREET_NUM = 179\\r\\n\\u003cbr\\u003e\\r\\nSTREET_SUF = DR\\r\\n\\u003cbr\\u003e\\r\\nUNIT_OR_LO = 2\\r\\n\\u003cbr\\u003e\\r\\nZIP_CODE = 65672\\r\\n\\u003cbr\\u003e\\r\\n","ResultHtml":"Address = 179 VILLA DR\\r\\n\\u003cbr\\u003e\\r\\nBLDING_SUI = \\r\\n\\u003cbr\\u003e\\r\\nCITY_LIMIT = TANEY COUNTY\\r\\n\\u003cbr\\u003e\\r\\nCOMMUNITY = HOLLISTER\\r\\n\\u003cbr\\u003e\\r\\ncreated_date = \\r\\n\\u003cbr\\u003e\\r\\ncreated_user = \\r\\n\\u003cbr\\u003e\\r\\nDATAFILE = A053003.ssf\\r\\n\\u003cbr\\u003e\\r\\nEASTING = 1408571.247\\r\\n\\u003cbr\\u003e\\r\\nEmail_Dist = \\r\\n\\u003cbr\\u003e\\r\\nESN = 12_166\\r\\n\\u003cbr\\u003e\\r\\nFEAT_NAME = Address_\\r\\n\\u003cbr\\u003e\\r\\nGPS_DATE = 1054252800000\\r\\n\\u003cbr\\u003e\\r\\nGPS_HEIGHT = 0\\r\\n\\u003cbr\\u003e\\r\\nGPS_TIME = 01:56:42pm\\r\\n\\u003cbr\\u003e\\r\\nlast_edited_date = 1490699094000\\r\\n\\u003cbr\\u003e\\r\\nlast_edited_user = ISGIS\\r\\n\\u003cbr\\u003e\\r\\nLat = NULL\\r\\n\\u003cbr\\u003e\\r\\nLong = NULL\\r\\n\\u003cbr\\u003e\\r\\nNORTHING = 265777.30800000002\\r\\n\\u003cbr\\u003e\\r\\nNOTES = \\r\\n\\u003cbr\\u003e\\r\\nNotes_2 = \\r\\n\\u003cbr\\u003e\\r\\nPoints_ID = 4\\r\\n\\u003cbr\\u003e\\r\\nPROPERTY_N = \\r\\n\\u003cbr\\u003e\\r\\nSTATE = MO\\r\\n\\u003cbr\\u003e\\r\\nStatus = \\r\\n\\u003cbr\\u003e\\r\\nSTREET_DI2 = \\r\\n\\u003cbr\\u003e\\r\\nSTREET_DIR = \\r\\n\\u003cbr\\u003e\\r\\nSTREET_NAM = VILLA\\r\\n\\u003cbr\\u003e\\r\\nSTREET_NUM = 179\\r\\n\\u003cbr\\u003e\\r\\nSTREET_SUF = DR\\r\\n\\u003cbr\\u003e\\r\\nUNIT_OR_LO = 2\\r\\n\\u003cbr\\u003e\\r\\nZIP_CODE = 65672\\r\\n\\u003cbr\\u003e\\r\\n","ParentId":null,"ChildIds":null}''', 9 | ) 10 | 11 | class TestDump (unittest.TestCase): 12 | 13 | def test_get_connection(self): 14 | ''' 15 | ''' 16 | with unittest.mock.patch('http.client.HTTPSConnection') as HTTPSConnection: 17 | conn, path = dump.get_connection('https://beacon.schneidercorp.com/api/beaconCore/GetVectorLayer?QPS=xxxx') 18 | 19 | HTTPSConnection.assert_called_once_with('beacon.schneidercorp.com') 20 | self.assertIs(conn, HTTPSConnection.return_value) 21 | self.assertEqual(path, '/api/beaconCore/GetVectorLayer?QPS=xxxx') 22 | 23 | def test_coordinate_pattern(self): 24 | ''' 25 | ''' 26 | wkts = [ 27 | 'point(123 456)', 28 | 'multipoint((123 456))', 29 | 'linestring(( 123 456 , 789 123))', 30 | 'point(123 -456)', 31 | 'point(123 -456.0)', 32 | 'point( -123 456 )', 33 | 'point(-123.0 456)', 34 | 'point(-123 -456)', 35 | 'point(-123.0 -456.0)', 36 | ] 37 | 38 | for wkt in wkts: 39 | match = dump.coordinate_pattern.search(wkt) 40 | self.assertEqual(abs(float(match.group('x'))), 123) 41 | self.assertEqual(abs(float(match.group('y'))), 456) 42 | 43 | def test_get_starting_bbox(self): 44 | ''' 45 | ''' 46 | conn, path = unittest.mock.Mock(), '/api/beaconCore/GetVectorLayer?QPS=xxxx' 47 | 48 | conn.getresponse.return_value.status = 200 49 | conn.getresponse.return_value.read.return_value = '''{"d":[{"Key":"1","Fields":{},"WktGeometry":"MULTIPOINT ((1408409 265912))","TipHtml":"Address = 160 VILLA DR\\r\\n\\u003cbr\\u003e\\r\\nBLDING_SUI = \\r\\n\\u003cbr\\u003e\\r\\nCITY_LIMIT = TANEY COUNTY\\r\\n\\u003cbr\\u003e\\r\\nCOMMUNITY = HOLLISTER\\r\\n\\u003cbr\\u003e\\r\\ncreated_date = \\r\\n\\u003cbr\\u003e\\r\\ncreated_user = \\r\\n\\u003cbr\\u003e\\r\\nDATAFILE = A053003.ssf\\r\\n\\u003cbr\\u003e\\r\\nEASTING = 1408483.1000000001\\r\\n\\u003cbr\\u003e\\r\\nEmail_Dist = \\r\\n\\u003cbr\\u003e\\r\\nESN = 12_166\\r\\n\\u003cbr\\u003e\\r\\nFEAT_NAME = Address_\\r\\n\\u003cbr\\u003e\\r\\nGPS_DATE = 1054252800000\\r\\n\\u003cbr\\u003e\\r\\nGPS_HEIGHT = 0\\r\\n\\u003cbr\\u003e\\r\\nGPS_TIME = 01:55:53pm\\r\\n\\u003cbr\\u003e\\r\\nlast_edited_date = 1490699094000\\r\\n\\u003cbr\\u003e\\r\\nlast_edited_user = ISGIS\\r\\n\\u003cbr\\u003e\\r\\nLat = 36.561191497240003\\r\\n\\u003cbr\\u003e\\r\\nLong = -93.289987614449998\\r\\n\\u003cbr\\u003e\\r\\nNORTHING = 265915.70400000003\\r\\n\\u003cbr\\u003e\\r\\nNOTES = \\r\\n\\u003cbr\\u003e\\r\\nNotes_2 = \\r\\n\\u003cbr\\u003e\\r\\nPoints_ID = 1\\r\\n\\u003cbr\\u003e\\r\\nPROPERTY_N = \\r\\n\\u003cbr\\u003e\\r\\nSTATE = MO\\r\\n\\u003cbr\\u003e\\r\\nStatus = \\r\\n\\u003cbr\\u003e\\r\\nSTREET_DI2 = \\r\\n\\u003cbr\\u003e\\r\\nSTREET_DIR = \\r\\n\\u003cbr\\u003e\\r\\nSTREET_NAM = VILLA\\r\\n\\u003cbr\\u003e\\r\\nSTREET_NUM = 160\\r\\n\\u003cbr\\u003e\\r\\nSTREET_SUF = DR\\r\\n\\u003cbr\\u003e\\r\\nUNIT_OR_LO = 2\\r\\n\\u003cbr\\u003e\\r\\nZIP_CODE = 65672\\r\\n\\u003cbr\\u003e\\r\\n","ResultHtml":"Address = 160 VILLA DR\\r\\n\\u003cbr\\u003e\\r\\nBLDING_SUI = \\r\\n\\u003cbr\\u003e\\r\\nCITY_LIMIT = TANEY COUNTY\\r\\n\\u003cbr\\u003e\\r\\nCOMMUNITY = HOLLISTER\\r\\n\\u003cbr\\u003e\\r\\ncreated_date = \\r\\n\\u003cbr\\u003e\\r\\ncreated_user = \\r\\n\\u003cbr\\u003e\\r\\nDATAFILE = A053003.ssf\\r\\n\\u003cbr\\u003e\\r\\nEASTING = 1408483.1000000001\\r\\n\\u003cbr\\u003e\\r\\nEmail_Dist = \\r\\n\\u003cbr\\u003e\\r\\nESN = 12_166\\r\\n\\u003cbr\\u003e\\r\\nFEAT_NAME = Address_\\r\\n\\u003cbr\\u003e\\r\\nGPS_DATE = 1054252800000\\r\\n\\u003cbr\\u003e\\r\\nGPS_HEIGHT = 0\\r\\n\\u003cbr\\u003e\\r\\nGPS_TIME = 01:55:53pm\\r\\n\\u003cbr\\u003e\\r\\nlast_edited_date = 1490699094000\\r\\n\\u003cbr\\u003e\\r\\nlast_edited_user = ISGIS\\r\\n\\u003cbr\\u003e\\r\\nLat = 36.561191497240003\\r\\n\\u003cbr\\u003e\\r\\nLong = -93.289987614449998\\r\\n\\u003cbr\\u003e\\r\\nNORTHING = 265915.70400000003\\r\\n\\u003cbr\\u003e\\r\\nNOTES = \\r\\n\\u003cbr\\u003e\\r\\nNotes_2 = \\r\\n\\u003cbr\\u003e\\r\\nPoints_ID = 1\\r\\n\\u003cbr\\u003e\\r\\nPROPERTY_N = \\r\\n\\u003cbr\\u003e\\r\\nSTATE = MO\\r\\n\\u003cbr\\u003e\\r\\nStatus = \\r\\n\\u003cbr\\u003e\\r\\nSTREET_DI2 = \\r\\n\\u003cbr\\u003e\\r\\nSTREET_DIR = \\r\\n\\u003cbr\\u003e\\r\\nSTREET_NAM = VILLA\\r\\n\\u003cbr\\u003e\\r\\nSTREET_NUM = 160\\r\\n\\u003cbr\\u003e\\r\\nSTREET_SUF = DR\\r\\n\\u003cbr\\u003e\\r\\nUNIT_OR_LO = 2\\r\\n\\u003cbr\\u003e\\r\\nZIP_CODE = 65672\\r\\n\\u003cbr\\u003e\\r\\n","ParentId":null,"ChildIds":null}]}''' 50 | 51 | bbox = dump.get_starting_bbox(conn, path, 13494, 100) 52 | args, kwargs = conn.request.mock_calls[-1][1:] 53 | 54 | self.assertEqual(args, ('POST', )) 55 | self.assertEqual(kwargs['url'], path) 56 | self.assertIn('"layerId": 13494', kwargs['body']) 57 | self.assertEqual(bbox, (1308409, 165912, 1508409, 365912)) 58 | 59 | def test_partition_bbox(self): 60 | ''' 61 | ''' 62 | boxes = dump.partition_bbox(0, 1, 2, 3) 63 | self.assertEqual(boxes, [(0, 1, 1, 2), (0, 2, 1, 3), (1, 1, 2, 2), (1, 2, 2, 3)]) 64 | 65 | def test_get_features1(self): 66 | ''' 67 | ''' 68 | conn = unittest.mock.Mock() 69 | path = '/api/beaconCore/GetVectorLayer?QPS=xxxx' 70 | bbox = (1308409, 165912, 1508409, 365912) 71 | bboxes = [bbox] + dump.partition_bbox(*bbox) 72 | 73 | conn.getresponse.return_value.status = 200 74 | responses = [ 75 | # top-level has all four records 76 | '''{{"d":[{F1},{F2},{F3},{F4}]}}'''.format(**Records), 77 | 78 | # first two recursed boxes have no records 79 | '''{"d":[]}''', 80 | '''{"d":[]}''', 81 | 82 | # third recursed box has three records 83 | '''{{"d":[{F1},{F2},{F3}]}}'''.format(**Records), 84 | 85 | # fourth recursed box has last record 86 | '''{{"d":[{F4}]}}'''.format(**Records), 87 | ] 88 | conn.getresponse.return_value.read.side_effect = lambda: responses.pop(0) 89 | 90 | features = dump.get_features(conn, path, 13494, bbox) 91 | ids = {f.get('id') for f in features} 92 | 93 | self.assertEqual(ids, {'1', '2', '3', '4'}) 94 | self.assertEqual(len(features), 4) 95 | self.assertEqual(len(conn.request.mock_calls), 5) 96 | 97 | for (_bbox, (_, args, kwargs)) in zip(bboxes, conn.request.mock_calls): 98 | body = json.loads(kwargs['body']) 99 | 100 | self.assertEqual(args, ('POST', )) 101 | self.assertEqual(kwargs['url'], path) 102 | self.assertEqual(body['layerId'], 13494) 103 | self.assertIn(body['featureLimit'], (0, 4)) 104 | self.assertEqual(body['ext'], { 105 | 'minx': _bbox[0], 'miny': _bbox[1], 'maxx': _bbox[2], 'maxy': _bbox[3] 106 | }) 107 | 108 | def test_get_features2(self): 109 | ''' 110 | ''' 111 | conn = unittest.mock.Mock() 112 | path = '/api/beaconCore/GetVectorLayer?QPS=xxxx' 113 | bbox = (1308409, 165912, 1508409, 365912) 114 | 115 | # splice in a level of recursion at the third sub-bbox 116 | bboxes_ = [bbox] + dump.partition_bbox(*bbox) 117 | bboxes = bboxes_[:4] + dump.partition_bbox(*bboxes_[3]) + bboxes_[4:] 118 | 119 | conn.getresponse.return_value.status = 200 120 | responses = [ 121 | # top-level has limit three records 122 | '''{{"d":[{F1},{F2},{F3}]}}'''.format(**Records), 123 | 124 | # first two recursed boxes have no records 125 | '''{"d":[]}''', 126 | '''{"d":[]}''', 127 | 128 | # third recursed box has limit three records 129 | '''{{"d":[{F1},{F2},{F3}]}}'''.format(**Records), 130 | 131 | # first two re-recursed boxes have no records 132 | '''{"d":[]}''', 133 | '''{"d":[]}''', 134 | 135 | # third re-recursed box has first two records 136 | '''{{"d":[{F1},{F2}]}}'''.format(**Records), 137 | 138 | # fourth re-recursed box has last two records 139 | '''{{"d":[{F3},{F4}]}}'''.format(**Records), 140 | 141 | # fourth recursed box has no records 142 | '''{"d":[]}''', 143 | ] 144 | conn.getresponse.return_value.read.side_effect = lambda: responses.pop(0) 145 | 146 | features = dump.get_features(conn, path, 13494, bbox) 147 | ids = {f.get('id') for f in features} 148 | 149 | self.assertEqual(ids, {'1', '2', '3', '4'}) 150 | self.assertEqual(len(features), 4) 151 | self.assertEqual(len(conn.request.mock_calls), 9) 152 | 153 | for (_bbox, (_, args, kwargs)) in zip(bboxes, conn.request.mock_calls): 154 | body = json.loads(kwargs['body']) 155 | 156 | self.assertEqual(args, ('POST', )) 157 | self.assertEqual(kwargs['url'], path) 158 | self.assertEqual(body['layerId'], 13494) 159 | self.assertIn(body['featureLimit'], (0, 3)) 160 | self.assertEqual(body['ext'], { 161 | 'minx': _bbox[0], 'miny': _bbox[1], 'maxx': _bbox[2], 'maxy': _bbox[3] 162 | }) 163 | 164 | def test_extract_properties(self): 165 | ''' 166 | ''' 167 | properties = dump.extract_properties(json.loads(Records['F1'])) 168 | 169 | self.assertEqual(properties, { 170 | 'Address': '160 VILLA DR', 'BLDING_SUI': ' ', 'CITY_LIMIT': 'TANEY COUNTY', 171 | 'COMMUNITY': 'HOLLISTER', 'created_date': '', 'created_user': '', 172 | 'DATAFILE': 'A053003.ssf', 'EASTING': '1408483.1000000001', 173 | 'Email_Dist': '', 'ESN': '12_166', 'FEAT_NAME': 'Address_', 174 | 'GPS_DATE': '1054252800000', 'GPS_HEIGHT': '0', 'GPS_TIME': '01:55:53pm', 175 | 'last_edited_date': '1490699094000', 'last_edited_user': 'ISGIS', 176 | 'Lat': '36.561191497240003', 'Long': '-93.289987614449998', 177 | 'NORTHING': '265915.70400000003', 'NOTES': ' ', 'Notes_2': '', 178 | 'Points_ID': '1', 'PROPERTY_N': '', 'STATE': 'MO', 'Status': '', 179 | 'STREET_DI2': ' ', 'STREET_DIR': ' ', 'STREET_NAM': 'VILLA', 180 | 'STREET_NUM': '160', 'STREET_SUF': 'DR', 'UNIT_OR_LO': '2', 181 | 'ZIP_CODE': '65672'}) 182 | 183 | def test_extract_geometry(self): 184 | ''' 185 | ''' 186 | geom1 = dump.extract_geometry(json.loads(Records['F1'])) 187 | geom2 = dump.extract_geometry(json.loads(Records['F2'])) 188 | geom3 = dump.extract_geometry(json.loads(Records['F3'])) 189 | geom4 = dump.extract_geometry(json.loads(Records['F4'])) 190 | 191 | self.assertEqual(geom1, {'type': 'Point', 192 | 'coordinates': [-93.289987614449998, 36.561191497240003]}) 193 | self.assertEqual(geom2, {'type': 'Point', 194 | 'coordinates': [-93.289737463319995, 36.560894620340001]}) 195 | self.assertEqual(geom3, {'type': 'Point', 196 | 'coordinates': [-93.289311785600006, 36.560464849970003]}) 197 | self.assertIsNone(geom4) 198 | 199 | def test_make_feature(self): 200 | ''' 201 | ''' 202 | feature = dump.make_feature(json.loads(Records['F1'])) 203 | 204 | self.assertEqual(feature, dict( 205 | type='Feature', id='1', 206 | geometry={ 207 | 'type': 'Point', 208 | 'coordinates': [-93.289987614449998, 36.561191497240003] 209 | }, 210 | properties={ 211 | 'Address': '160 VILLA DR', 'BLDING_SUI': ' ', 'CITY_LIMIT': 'TANEY COUNTY', 212 | 'COMMUNITY': 'HOLLISTER', 'created_date': '', 'created_user': '', 213 | 'DATAFILE': 'A053003.ssf', 'EASTING': '1408483.1000000001', 214 | 'Email_Dist': '', 'ESN': '12_166', 'FEAT_NAME': 'Address_', 215 | 'GPS_DATE': '1054252800000', 'GPS_HEIGHT': '0', 'GPS_TIME': '01:55:53pm', 216 | 'last_edited_date': '1490699094000', 'last_edited_user': 'ISGIS', 217 | 'Lat': '36.561191497240003', 'Long': '-93.289987614449998', 218 | 'NORTHING': '265915.70400000003', 'NOTES': ' ', 'Notes_2': '', 219 | 'Points_ID': '1', 'PROPERTY_N': '', 'STATE': 'MO', 'Status': '', 220 | 'STREET_DI2': ' ', 'STREET_DIR': ' ', 'STREET_NAM': 'VILLA', 221 | 'STREET_NUM': '160', 'STREET_SUF': 'DR', 'UNIT_OR_LO': '2', 222 | 'ZIP_CODE': '65672' 223 | })) 224 | 225 | if __name__ == '__main__': 226 | unittest.main() --------------------------------------------------------------------------------