├── .gitignore ├── README.md ├── accessibility.sql ├── accessibility.tsv ├── const.py ├── ctx.py ├── halte-db.py ├── indices.sql ├── kv7-db-auto.py ├── kv7-db.py ├── kv78turbo-api.py ├── kv78turbo-client.py ├── nsavt.py ├── rrd.py ├── sphinx.conf ├── sphinxapi.py └── wordforms.txt /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[co] 2 | 3 | # Packages 4 | *.egg 5 | *.egg-info 6 | dist 7 | build 8 | eggs 9 | parts 10 | bin 11 | var 12 | sdist 13 | develop-eggs 14 | .installed.cfg 15 | 16 | # Installer logs 17 | pip-log.txt 18 | 19 | # Unit test / coverage reports 20 | .coverage 21 | .tox 22 | 23 | #Translations 24 | *.mo 25 | 26 | #Mr Developer 27 | .mr.developer.cfg 28 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | KV78Turbo-OVAPI 2 | =============== -------------------------------------------------------------------------------- /accessibility.sql: -------------------------------------------------------------------------------- 1 | drop table accessibility; 2 | 3 | CREATE TABLE "accessibility" ( 4 | "timingpointcode" VARCHAR(10) NOT NULL, 5 | "timingpointname" VARCHAR(50) NOT NULL, 6 | "wheelchairaccessible" VARCHAR(13) NOT NULL, 7 | "visualaccessible" VARCHAR(13) NOT NULL, 8 | PRIMARY KEY ("timingpointcode") 9 | ); 10 | copy accessibility from '/tmp/accessibility.tsv' delimiter '|' CSV; 11 | alter table timingpoint add column wheelchairaccessible VARCHAR(13); 12 | alter table timingpoint add column visualaccessible VARCHAR(13); 13 | 14 | update timingpoint set wheelchairaccessible = 'ACCESSIBLE' WHERE timingpointcode in 15 | (SELECT timingpointcode FROM accessibility WHERE wheelchairaccessible = 'ACCESSIBLE'); 16 | 17 | update timingpoint set wheelchairaccessible = 'NOTACCESSIBLE' WHERE timingpointcode in 18 | (SELECT timingpointcode FROM accessibility WHERE wheelchairaccessible = 'NOTACCESSIBLE'); 19 | 20 | update timingpoint set visualaccessible = 'ACCESSIBLE' WHERE timingpointcode in 21 | (SELECT timingpointcode FROM accessibility WHERE visualaccessible = 'ACCESSIBLE'); 22 | 23 | update timingpoint set visualaccessible = 'NOTACCESSIBLE' WHERE timingpointcode in 24 | (SELECT timingpointcode FROM accessibility WHERE visualaccessible = 'NOTACCESSIBLE'); 25 | -------------------------------------------------------------------------------- /const.py: -------------------------------------------------------------------------------- 1 | ZMQ_KV8 = "tcp://kv7.openov.nl:7817" 2 | ZMQ_KV78UWSGI = "tcp://127.0.0.1:6090" 3 | ZMQ_KV7 = "tcp://127.0.0.1:6070" 4 | -------------------------------------------------------------------------------- /ctx.py: -------------------------------------------------------------------------------- 1 | # TODO: this class needs to handle escaped '|' 2 | 3 | import simplejson 4 | 5 | class ctxtable: 6 | def __init__(self, Label, Name = None, Comment = None, Path = None, Endian = None, Enc = None, Res1 = None): 7 | self.Label = Label 8 | self.Name = Name 9 | self.Comment = Comment 10 | self.Path = Path 11 | self.Endian = Endian 12 | self.Enc = Enc 13 | self.Res1 = Res1 14 | 15 | self.fieldlabels = [] 16 | self.values = [] 17 | 18 | def set_fieldlabels(self, fieldlabels): 19 | self.fieldlabels = fieldlabels 20 | 21 | def dict(self): 22 | return {'meta': {'label': self.Label, 'name': self.Name, 'comment': self.Comment, 'path': self.Path, 'endian': self.Endian, 'enc': self.Enc, 'res1': self.Res1}, 'fieldlabels': self.fieldlabels, 'values': self.values} 23 | 24 | def columns(self): 25 | result = {} 26 | for x in range(0, len(self.fieldlabels)): 27 | result[self.fieldlabels[x]] = [y[x] for y in self.values] 28 | 29 | return result 30 | 31 | def rows(self, empty=False): 32 | results = [] 33 | for y in self.values: 34 | result = {} 35 | for x in range(0, len(self.fieldlabels)): 36 | if not empty or y[x] is not None: 37 | result[self.fieldlabels[x]] = y[x] 38 | if len(result) > 0: 39 | results.append(result) 40 | return results 41 | 42 | def rowsdict(self, keys): 43 | results = {} 44 | for y in self.values: 45 | result = {} 46 | for x in range(0, len(self.fieldlabels)): 47 | if y[x] is not None: 48 | result[self.fieldlabels[x]] = y[x] 49 | if len(result) > 0: 50 | key = '|'.join([result[x] for x in keys]) 51 | results[key] = result 52 | return results 53 | 54 | def append(self, result): 55 | tmp = [] 56 | for x in result: 57 | if x == '\\0': 58 | tmp.append(None) 59 | else: 60 | tmp.append(intern(x)) 61 | 62 | self.values.append(tmp) 63 | 64 | def __repr__(self): 65 | return str(self.Label) 66 | 67 | class ctx: 68 | def __init__(self, content = None): 69 | self.ctx = {} 70 | if content is not None: 71 | self._content = content 72 | self.parse() 73 | 74 | def parse(self): 75 | tLabel = None 76 | 77 | for line in self._content.split('\r\n')[:-1]: 78 | if line[0] == '\\': 79 | # control characters 80 | if line[1] == 'G': 81 | Label, Name, Comment, Path, Endian, Enc, Res1, TimeStamp, _ = line[2:].split('|') 82 | self.ctx['Subscription'] = Comment 83 | elif line[1] == 'T': 84 | tmp = ctxtable(*line[2:].split('|')) 85 | tLabel = tmp.Label 86 | self.ctx[intern(tLabel)] = tmp 87 | elif line[1] == 'L': 88 | self.ctx[tLabel].set_fieldlabels(line[2:].split('|')) 89 | 90 | else: 91 | self.ctx[tLabel].append(line.split('|')) 92 | 93 | 94 | #contents = open('/tmp/KV8/1331344996.79').read() 95 | #c = ctx(contents) 96 | #print c.ctx['DATEDPASSTIME'].rows() 97 | -------------------------------------------------------------------------------- /halte-db.py: -------------------------------------------------------------------------------- 1 | import uwsgi 2 | import psycopg2 3 | from psycopg2.pool import PersistentConnectionPool 4 | import simplejson 5 | from sphinxapi import * 6 | from urlparse import urlparse,urlsplit,parse_qs 7 | import re 8 | 9 | COMMON_HEADERS = [('Content-Type', 'application/json'), ('Access-Control-Allow-Origin', '*'), ('Access-Control-Allow-Headers', 'Requested-With,Content-Type')] 10 | 11 | pool = PersistentConnectionPool(1,20,"dbname='haltes'") 12 | 13 | #update timingpoint set latitude = CAST(ST_Y(the_geom) AS NUMERIC(9,7)), longitude = CAST(ST_X(the_geom) AS NUMERIC(8,7)) FROM (select ST_Transform(st_setsrid(st_makepoint(locationx_ew, locationy_ns), 28992), 4326) AS the_geom from timingpoint as t2 where t2.timingpointcode = timingpointcode) AS W; 14 | 15 | def notfound(start_response): 16 | start_response('404 File Not Found', COMMON_HEADERS + [('Content-length', '2')]) 17 | yield '[]' 18 | 19 | def searchStops(query): 20 | reply = {'Columns' : ['TimingPointTown','TimingPointName', 'Name', 'Latitude', 'Longitude'] , 'Rows' : []} 21 | if query == 'search_suggest_query' or query == None or query == '' or len(query) < 2: 22 | return reply 23 | cl = SphinxClient() 24 | cl.SetServer('localhost', 9312) 25 | cl.SetWeights ( [100, 1] ) 26 | cl.SetMatchMode ( SPH_MATCH_EXTENDED ) 27 | cl.SetIndexWeights({'haltes' : 10,'haltesstemmed' : 1,'haltes_metaphone' : 5}) 28 | cl.SetFieldWeights({'name' : 20, 'timingpointtown' : 10, 'timingpointname' : 1, 'namespaceless' : 4, 'timingpointtownspaceless' : 2, 'timingpointnamespaceless' : 1,'stopareacode' :1}) 29 | cl.Open() 30 | try: 31 | res = cl.Query ( query, 'haltes haltesstemmed haltes_metaphone' ) 32 | if res.has_key('matches'): 33 | for match in res['matches']: 34 | row = {} 35 | for attr in res['attrs']: 36 | attrname = attr[0] 37 | value = match['attrs'][attrname] 38 | row[attrname] = value 39 | reply['Rows'].append([row['timingpointtown'],row['timingpointname'],row['name'],row['latitude'],row['longitude']]) 40 | except Exception as e: 41 | print e 42 | pass 43 | cl.Close() 44 | return reply 45 | 46 | def search(conn,environ, start_response): 47 | params = parse_qs(environ.get('QUERY_STRING','')) 48 | reply = {'Columns' : ['TimingPointTown','TimingPointName', 'Name', 'Latitude','Longitude'], 'Rows' : []} 49 | params = parse_qs(environ.get('QUERY_STRING','')) 50 | query = params['query'][0] 51 | if query == 'search_suggest_query' or query == None or query == '' or len(query) < 2: 52 | return reply 53 | cl = SphinxClient() 54 | cl.SetServer('localhost', 9312) 55 | cl.SetWeights ( [100, 1] ) 56 | if 'sgh' in params: 57 | cl.SetFilter('sgh',[1]) 58 | cl.SetMatchMode ( SPH_MATCH_EXTENDED ) 59 | cl.SetIndexWeights({'haltes' : 10,'haltesstemmed' : 1,'haltes_metaphone' : 5}) 60 | cl.SetFieldWeights({'name' : 20, 'timingpointtown' : 10, 'timingpointname' : 1, 'namespaceless' : 4, 'timingpointtownspaceless' : 2, 'timingpointnamespaceless' : 1,'stopareacode' :1}) 61 | cl.Open() 62 | try: 63 | res = cl.Query ( query, 'haltes haltesstemmed haltes_metaphone' ) 64 | if res.has_key('matches'): 65 | for match in res['matches']: 66 | row = {} 67 | for attr in res['attrs']: 68 | attrname = attr[0] 69 | value = match['attrs'][attrname] 70 | row[attrname] = value 71 | reply['Rows'].append([row['timingpointtown'],row['timingpointname'],row['name'],row['latitude'],row['longitude']]) 72 | except Exception as e: 73 | print e 74 | pass 75 | cl.Close() 76 | return reply 77 | 78 | 79 | townreply = {'Columns' : ['TimingPointTown'], 'Rows' : []} 80 | conn = pool.getconn() 81 | cur = conn.cursor() 82 | cur.execute("SELECT distinct timingpointtown FROM timingpoint ORDER BY timingpointtown", []) 83 | rows = cur.fetchall() 84 | for row in rows: 85 | townreply['Rows'].append([row[0]]) 86 | cur.close() 87 | pool.putconn(conn) 88 | 89 | def queryTowns(conn,environ, start_response): 90 | params = parse_qs(environ.get('QUERY_STRING','')) 91 | if 'sgh' in params: 92 | reply = {'Columns' : ['TimingPointTown'], 'Rows' : []} 93 | cur = conn.cursor() 94 | cur.execute("SELECT distinct timingpointtown FROM timingpoint where sgh = true ORDER BY timingpointtown", []) 95 | rows = cur.fetchall() 96 | for row in rows: 97 | reply['Rows'].append([row[0]]) 98 | cur.close() 99 | return reply 100 | else: 101 | return townreply 102 | 103 | def queryStops(conn,environ, start_response): 104 | params = parse_qs(environ.get('QUERY_STRING','')) 105 | reply = {'Columns' : ['TimingPointTown', 'Name', 'Latitude', 'Longitude' , 'StopAreaCode'] , 'Rows' : []} 106 | cur = conn.cursor() 107 | if 'town' in params: 108 | cur.execute("SELECT distinct on (timingpointtown,name) timingpointtown,name,latitude,longitude,stopareacode FROM timingpoint WHERE timingpointtown = %s ORDER BY name", [params['town'][0]]) 109 | elif 'tpc' in params: 110 | cur.execute("SELECT distinct on (timingpointtown,name) timingpointtown,name,latitude,longitude,stopareacode FROM timingpoint WHERE timingpointcode = %s ORDER BY name", [params['tpc'][0]]) 111 | elif 'bottomright' in params and 'topleft' in params and 'sgh' in params: 112 | minLatitude, maxLongitude = params['bottomright'][0].split(',') 113 | maxLatitude, minLongitude = params['topleft'][0].split(',') 114 | cur.execute("SELECT distinct on (timingpointtown,name) timingpointtown,name,avg(latitude),avg(longitude),stopareacode FROM timingpoint WHERE latitude between %s AND %s AND longitude between %s and %s and sgh = true group by timingpointtown,name,stopareacode", [minLatitude,maxLatitude,minLongitude,maxLongitude]) 115 | elif 'bottomright' in params and 'topleft' in params: 116 | minLatitude, maxLongitude = params['bottomright'][0].split(',') 117 | maxLatitude, minLongitude = params['topleft'][0].split(',') 118 | cur.execute("SELECT distinct on (timingpointtown,name) timingpointtown,name,avg(latitude),avg(longitude),stopareacode FROM timingpoint WHERE latitude between %s AND %s AND longitude between %s and %s group by timingpointtown,name,stopareacode", [minLatitude,maxLatitude,minLongitude,maxLongitude]) 119 | elif 'near' in params and 'sgh' in params: 120 | latitude, longitude = params['near'][0].split(',') 121 | limit = '100' 122 | if 'limit' in params: 123 | limit = params['limit'][0] 124 | cur = conn.cursor() 125 | geomconstant = 'SRID=4326;POINT('+longitude+' '+latitude+')' 126 | cur.execute("SELECT timingpointtown,name,latitude,longitude,stopareacode FROM (SELECT distinct on (timingpointtown,name) timingpointtown,name,latitude,longitude,stopareacode,distance FROM (select timingpointtown,name,latitude,longitude,stopareacode, ST_Distance(the_geom, st_setsrid(st_makepoint(%s, %s),4326)) as distance from timingpoint where sgh = true ORDER by the_geom <-> %s::geometry LIMIT %s) as x) as y order by distance;", [longitude,latitude,geomconstant,limit]) 127 | elif 'near' in params: 128 | latitude, longitude = params['near'][0].split(',') 129 | limit = '100' 130 | if 'limit' in params: 131 | limit = params['limit'][0] 132 | cur = conn.cursor() 133 | geomconstant = 'SRID=4326;POINT('+longitude+' '+latitude+')' 134 | cur.execute("SELECT timingpointtown,name,latitude,longitude,stopareacode FROM (SELECT distinct on (timingpointtown,name) timingpointtown,name,latitude,longitude,stopareacode,distance FROM (select timingpointtown,name,latitude,longitude,stopareacode, ST_Distance(the_geom, st_setsrid(st_makepoint(%s, %s),4326)) as distance from timingpoint ORDER by the_geom <-> %s::geometry LIMIT %s) as x) as y order by distance;", [longitude,latitude,geomconstant,limit]) 135 | elif 'search' in params: 136 | return searchStops(params['search'][0]) 137 | else: 138 | return '404' 139 | rows = cur.fetchall() 140 | for row in rows: 141 | if len(reply['Columns']) == 5: 142 | reply['Rows'].append([row[0],row[1],row[2],row[3],row[4]]) 143 | if len(reply['Columns']) == 6: 144 | reply['Rows'].append([row[0],row[1],row[2],row[3],row[4],row[5]]) 145 | cur.close() 146 | return reply 147 | 148 | def queryStopAreas(conn,environ, start_response): 149 | params = parse_qs(environ.get('QUERY_STRING','')) 150 | reply = {'Columns' : ['TimingPointTown','TimingPointName','Name', 'Latitude', 'Longitude', 'StopAreaCode'] , 'Rows' : []} 151 | cur = conn.cursor() 152 | if 'near' in params: 153 | latitude, longitude = params['near'][0].split(',') 154 | limit = '100' 155 | if 'limit' in params: 156 | limit = params['limit'][0] 157 | limit5 = int(limit)*5 158 | cur = conn.cursor() 159 | geomconstant = 'SRID=4326;POINT('+longitude+' '+latitude+')' 160 | cur.execute("SELECT timingpointtown,timingpointname,name,latitude,longitude,stopareacode FROM (select distinct on (stopareacode) * from (select *, ST_Distance(the_geom, st_setsrid(st_makepoint(%s, %s),4326)) as distance from timingpoint ORDER by the_geom <-> %s::geometry LIMIT %s) as x) as y order by distance LIMIT %s;", [latitude,longitude,geomconstant,limit5,limit]) 161 | else: 162 | return '404' 163 | rows = cur.fetchall() 164 | for row in rows: 165 | reply['Rows'].append([row[0],row[1],row[2],row[3],row[4],row[5]]) 166 | cur.close() 167 | return reply 168 | 169 | def queryAccessibility(conn,environ, start_response): 170 | params = parse_qs(environ.get('QUERY_STRING','')) 171 | reply = {'Columns' : ['TimingPointTown', 'TimingPointName', 'Name', 'TimingPointCode', 'kv78turbo','TimingPointWheelChairAccessible', 'TimingPointVisualAccessible', 'Steps','Latitude', 'Longitude'] , 'Rows' : []} 172 | cur = conn.cursor() 173 | if 'tpc' in params: 174 | cur.execute("SELECT timingpointtown, timingpointname,name,t.timingpointcode,kv78turbo,motorisch,visueel,trap,t.latitude,t.longitude FROM timingpoint as t left join haltescan as h on (t.timingpointcode = h.timingpointcode) WHERE t.timingpointcode = %s", [params['tpc'][0]]) 175 | else: 176 | return '404' 177 | rows = cur.fetchall() 178 | for row in rows: 179 | reply['Rows'].append([row[0],row[1],row[2],row[3],row[4],row[5],row[6],row[7],row[8],row[9]]) 180 | cur.close() 181 | return reply 182 | 183 | def queryTimingPoints(conn,environ, start_response): 184 | params = parse_qs(environ.get('QUERY_STRING','')) 185 | reply = {'Columns' : ['TimingPointTown', 'TimingPointName', 'Name', 'TimingPointCode', 'StopAreaCode', 'kv55', 'kv78turbo', 'arriva55'] , 'Rows' : []} 186 | cur = conn.cursor() 187 | if 'town' in params and 'timingpointname' in params: 188 | cur.execute("SELECT timingpointtown,timingpointname,name,timingpointcode,stopareacode, kv55, kv78turbo, arriva55 FROM timingpoint WHERE timingpointtown = %s AND timingpoointname = %s", [params['town'][0], params['timingpointname'][0]]) 189 | elif 'town' in params and 'name' in params: 190 | cur.execute("SELECT timingpointtown,timingpointname,name,timingpointcode,stopareacode, kv55, kv78turbo, arriva55 FROM timingpoint WHERE timingpointtown = %s AND name = %s", [params['town'][0], params['name'][0]]) 191 | elif 'town' in params: 192 | cur.execute("SELECT timingpointtown,timingpointname,name,timingpointcode,stopareacode, kv55, kv78turbo, arriva55 FROM timingpoint WHERE timingpointtown = %s", [params['town'][0]]) 193 | elif 'tpc' in params: 194 | cur.execute("SELECT timingpointtown,timingpointname,name,timingpointcode,stopareacode, kv55, kv78turbo, arriva55 FROM timingpoint AS t1 WHERE EXISTS (select 1 FROM timingpoint AS t2 WHERE timingpointcode = %s and t1.name = t2.name AND t1.timingpointtown = t2.timingpointtown)", [params['tpc'][0]]) 195 | elif 'near' in params: 196 | latitude, longitude = params['near'][0].split(',') 197 | limit = '150' 198 | if 'limit' in params: 199 | limit = params['limit'][0] 200 | geomconstant = 'SRID=4326;POINT('+longitude+' '+latitude+')' 201 | cur = conn.cursor() 202 | if 'destinations' in params and 'accessibility' in params: 203 | reply['Columns'].extend(['DestinationName50','LinePlanningNumber','LinePublicNumber','TimingPointWheelChairAccessible','TimingPointVisualAccessible','Latitude','Longitude']) 204 | cur.execute("SELECT timingpointtown, timingpointname, name,x.timingpointcode,stopareacode,kv55,kv78turbo,arriva55,destinationname50,lineplanningnumber,linepublicnumber,motorisch,visueel,x.latitude,x.longitude from (SELECT latitude,longitude, timingpointtown,timingpointname,name,t.timingpointcode,stopareacode, kv55, kv78turbo, arriva55,destinationname50,lineplanningnumber,linepublicnumber, the_geom FROM (select * from timingpoint ORDER by the_geom <-> %s::geometry LIMIT %s) as t,destinationuserstop as d where d.timingpointcode = t.timingpointcode and destinationcode is not null) as x left join haltescan as h on (h.timingpointcode = x.timingpointcode) ORDER by the_geom <-> %s::geometry LIMIT %s;", [geomconstant,limit,geomconstant,limit]) 205 | elif 'destinations' in params: 206 | reply['Columns'].extend(['DestinationName50','LinePlanningNumber','LinePublicNumber','Latitude','Longitude']) 207 | cur.execute("SELECT timingpointtown,timingpointname,name,t.timingpointcode,stopareacode, kv55, kv78turbo, arriva55,destinationname50,lineplanningnumber,linepublicnumber,latitude,longitude FROM (select * from timingpoint ORDER by the_geom <-> %s::geometry LIMIT %s) as t,destinationuserstop as d where d.timingpointcode = t.timingpointcode and destinationcode is not null ORDER by the_geom <-> %s::geometry LIMIT %s;", [geomconstant,limit,geomconstant,limit]) 208 | else: 209 | cur.execute("SELECT timingpointtown,timingpointname,name,timingpointcode,stopareacode, kv55, kv78turbo, arriva55 FROM timingpoint ORDER by the_geom <-> %s::geometry LIMIT %s;", [geomconstant,limit]) 210 | else: 211 | return '404' 212 | rows = cur.fetchall() 213 | for row in rows: 214 | if len(reply['Columns']) == 8: 215 | reply['Rows'].append([row[0],row[1],row[2],row[3],row[4],row[5],row[6],row[7]]) 216 | elif len(reply['Columns']) == 13: 217 | reply['Rows'].append([row[0],row[1],row[2],row[3],row[4],row[5],row[6],row[7],row[8],row[9],row[10],row[11],row[12]]) 218 | elif len(reply['Columns']) == 15: 219 | reply['Rows'].append([row[0],row[1],row[2],row[3],row[4],row[5],row[6],row[7],row[8],row[9],row[10],row[11],row[12],row[13],row[14]]) 220 | cur.close() 221 | return reply 222 | 223 | def HalteDB(environ, start_response): 224 | url = environ['PATH_INFO'][1:] 225 | if len(url) > 0 and url[-1] == '/': 226 | url = url[:-1] 227 | arguments = url.split('/') 228 | reply = None 229 | try: 230 | conn = pool.getconn() 231 | if arguments[0] == 'towns': 232 | reply = queryTowns(conn,environ, start_response) 233 | elif arguments[0] == 'stops': 234 | reply = queryStops(conn,environ, start_response) 235 | elif arguments[0] == 'stopareas': 236 | reply = queryStopAreas(conn,environ, start_response) 237 | elif arguments[0] == 'timingpoints': 238 | reply = queryTimingPoints(conn,environ, start_response) 239 | elif arguments[0] == 'accessibility': 240 | reply = queryAccessibility(conn,environ, start_response) 241 | elif arguments[0] == 'search': 242 | reply = search(conn,environ, start_response) 243 | else: 244 | pool.putconn(conn) 245 | return notfound(start_response) 246 | pool.putconn(conn) 247 | except Exception as e: 248 | print e 249 | pool.putconn(conn,close=True) 250 | if reply == '404': 251 | return notfound(start_response) 252 | reply = simplejson.dumps(reply) 253 | start_response('200 OK', COMMON_HEADERS + [('Content-length', str(len(reply)))]) 254 | return reply 255 | 256 | uwsgi.applications = {'': HalteDB} 257 | -------------------------------------------------------------------------------- /indices.sql: -------------------------------------------------------------------------------- 1 | CREATE INDEX ON localservicegrouppasstime("targetarrivaltime"); 2 | CREATE INDEX ON localservicegrouppasstime("targetarrivaltime","localservicelevelcode"); 3 | CREATE INDEX ON localservicegrouppasstime(dataownercode,localservicelevelcode,lineplanningnumber,journeynumber,fortifyordernumber); 4 | CREATE INDEX ON localservicegroupvalidity("dataownercode","localservicelevelcode"); 5 | -------------------------------------------------------------------------------- /kv7-db-auto.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import zmq 3 | import simplejson as serializer 4 | import time 5 | from ctx import ctx 6 | from gzip import GzipFile 7 | from cStringIO import StringIO 8 | import psycopg2 9 | import time 10 | from datetime import datetime, timedelta 11 | from const import ZMQ_KV7 12 | from twisted.internet import task 13 | from twisted.internet import reactor 14 | import codecs 15 | import os 16 | 17 | sys.stderr.write('Setting up a ZeroMQ PUSH: %s\n' % (ZMQ_KV7)) 18 | context = zmq.Context() 19 | push = context.socket(zmq.PUSH) 20 | push.connect(ZMQ_KV7) 21 | variables = {} 22 | 23 | MINUTES_FORWARD = 90 24 | 25 | def secondsFromMidnight(time): 26 | hours, minutes, seconds = time.split(':') 27 | return (int(hours)*60*60) + (int(minutes)*60) + int(seconds) 28 | 29 | def time(seconds): 30 | hours = seconds / 3600 31 | if hours < 0: 32 | hours += 24 33 | seconds -= 3600*hours 34 | minutes = seconds / 60 35 | if minutes < 0: 36 | minutes *= -1 37 | seconds -= 60*minutes 38 | return "%02d:%02d:%02d" % (hours, minutes, seconds) 39 | 40 | now = datetime.now() + timedelta(minutes=MINUTES_FORWARD) - timedelta(seconds=120) 41 | 42 | def broadcastmeta(dbname): 43 | print 'Fetching meta' 44 | conn = psycopg2.connect("dbname='"+dbname+"'") 45 | cur = conn.cursor() 46 | cur.execute("SELECT dataownercode,lineplanningnumber,linepublicnumber,linename,transporttype from line", []) 47 | meta = {'LINE' : {}, 'DESTINATION' : {}, 'TIMINGPOINT' : {}} 48 | rows = cur.fetchall() 49 | for row in rows: 50 | line_id = intern(row[0] + '_' + row[1]) 51 | meta['LINE'][line_id] = {'LinePublicNumber' : intern(row[2]), 'LineName' : intern(row[3]), 'TransportType' : intern(row[4])} 52 | cur.close() 53 | 54 | cur = conn.cursor() 55 | cur.execute("SELECT dataownercode,destinationcode,destinationname50 from destination", []) 56 | rows = cur.fetchall() 57 | for row in rows: 58 | destination_id = intern(row[0] + '_' + row[1]) 59 | meta['DESTINATION'][destination_id] = intern(row[2]) 60 | cur.close() 61 | 62 | cur = conn.cursor() 63 | cur.execute(""" 64 | SELECT timingpointcode,timingpointname,timingpointtown,stopareacode,ST_Y(the_geom)::NUMERIC(9,7) AS lat,ST_X(the_geom)::NUMERIC(8,7) AS lon 65 | FROM 66 | (SELECT DISTINCT dataownercode,userstopcode FROM localservicegrouppasstime WHERE journeystoptype != 'INFOPOINT') as u 67 | JOIN usertimingpoint as ut USING (dataownercode,userstopcode) 68 | JOIN (select *,ST_Transform(st_setsrid(st_makepoint(coalesce(locationx_ew,0), coalesce(locationy_ns,0)), 28992), 4326) AS the_geom FROM timingpoint) as t USING (timingpointcode) 69 | """,[]) 70 | kv7rows = cur.fetchall() 71 | for kv7row in kv7rows: 72 | meta['TIMINGPOINT'][intern(kv7row[0])] = {'TimingPointName' : intern(kv7row[1]), 'TimingPointTown' : intern(kv7row[2]), 'StopAreaCode' : kv7row[3], 'Latitude' : float(kv7row[4]), 'Longitude' : float(kv7row[5])} 73 | #if kv7row[7] != 'UNKNOWN' and kv7row[7] is not None: 74 | # meta['TIMINGPOINT'][kv7row[0]]['TimingPointWheelChairAccessible'] = kv7row[7] 75 | #if kv7row[6] != 'UNKNOWN' and kv7row[6] is not None: 76 | # meta['TIMINGPOINT'][kv7row[0]]['TimingPointVisualAccessible'] = kv7row[6] 77 | push.send_json(meta) 78 | cur.close() 79 | conn.close() 80 | print 'meta refreshed' 81 | 82 | def currentdatabase(): 83 | output = codecs.open('/var/ovapi/kv7.openov.nl/GOVI/CURRENTDB', 'r', 'UTF-8') 84 | newdb = output.read().split('\n')[0] 85 | if 'lastdb' not in variables or variables['lastdb'] != newdb: 86 | broadcastmeta(newdb) 87 | variables['lastdb'] = newdb 88 | return newdb 89 | 90 | def fetchandpushkv7(): 91 | try: 92 | fetchandpush() 93 | except Exception as e: 94 | print e 95 | 96 | def fetchandpush(): 97 | dbname = currentdatabase() 98 | print 'Using database : ' + dbname 99 | conn = psycopg2.connect("dbname='"+dbname+"'") 100 | global now # this has to be done better 101 | now += timedelta(seconds=60) 102 | startrange = now.strftime("%H:%M:00") 103 | startdate = now.strftime("%Y-%m-%d") 104 | endrange = (datetime.now() + timedelta(minutes=MINUTES_FORWARD)).strftime("%H:%M:00") 105 | now = (datetime.now() + timedelta(minutes=MINUTES_FORWARD) - timedelta(minutes=1)) 106 | startdate48 = ((now + timedelta(seconds=60))-timedelta(days=1)).strftime("%Y-%m-%d") 107 | if endrange == '00:00:00': 108 | endrange = '24:00:00' 109 | shours,sminutes,sseconds = startrange.split(':') 110 | ehours,eminutes,eseconds = endrange.split(':') 111 | startrange48 = str(int(shours)+24) + ':' + sminutes + ':00' 112 | endrange48 = str(int(ehours)+24) + ':' + eminutes + ':00' 113 | sys.stdout.write(startrange + '-' + endrange + '@ ' + startdate) 114 | sys.stdout.write(' ') 115 | sys.stdout.write(startrange48 + '-' + endrange48 + '@ ' +startdate48 + '\n') 116 | cur = conn.cursor() 117 | cur.execute(""" 118 | select 119 | p.dataownercode,p.localservicelevelcode,p.lineplanningnumber,journeynumber,fortifyordernumber,p.userstopcode,userstopordernumber,linedirection, 120 | 121 | p.destinationcode,targetarrivaltime,targetdeparturetime,sidecode,wheelchairaccessible,journeystoptype,istimingstop,productformulatype,timingpointcode, 122 | timingpointdataownercode,operationdate ,journeypatterncode 123 | from localservicegrouppasstime as p, usertimingpoint as u, localservicegroupvalidity as v 124 | where exists ( 125 | SELECT 1 126 | FROM localservicegrouppasstime AS f, localservicegroupvalidity as v 127 | WHERE f.journeystoptype = 'FIRST' AND f.dataownercode = p.dataownercode 128 | AND f.localservicelevelcode = p.localservicelevelcode AND f.lineplanningnumber = p.lineplanningnumber 129 | and f.journeynumber = p.journeynumber AND f.fortifyordernumber = p.fortifyordernumber 130 | AND ( 131 | (operationdate = date %s AND targetarrivaltime >= %s AND targetarrivaltime < %s ) 132 | OR (operationdate = date %s AND targetarrivaltime >= %s AND targetarrivaltime < %s) 133 | ) 134 | AND f.localservicelevelcode = v.localservicelevelcode AND f.dataownercode = v.dataownercode 135 | ) 136 | AND p.dataownercode = u.dataownercode AND p.userstopcode = u.userstopcode AND journeystoptype != 'INFOPOINT' 137 | AND p.localservicelevelcode = v.localservicelevelcode AND p.dataownercode = v.dataownercode 138 | AND ( 139 | (operationdate = date %s AND targetarrivaltime >= %s and targetarrivaltime < %s) 140 | OR (operationdate = date %s AND targetarrivaltime >= %s) 141 | ) 142 | """, [startdate, startrange,endrange,startdate48,startrange48,endrange48,startdate,startrange,startrange48,startdate48,startrange48]) 143 | kv7rows = cur.fetchall() 144 | passes = {'PASSTIMES' : {} } 145 | updatetimestamp = datetime.today().strftime("%Y-%m-%dT%H:%M:%S") + "+02:00" 146 | print str(len(kv7rows)) + ' rows from db' 147 | for kv7row in kv7rows: 148 | row = {} 149 | row['DataOwnerCode'] = kv7row[0] 150 | row['LocalServiceLevelCode'] = kv7row[1] 151 | row['LinePlanningNumber'] = kv7row[2] 152 | row['JourneyNumber'] = kv7row[3] 153 | row['FortifyOrderNumber'] = kv7row[4] 154 | row['UserStopCode'] = kv7row[5] 155 | row['UserStopOrderNumber'] = kv7row[6] 156 | row['LineDirection'] = int(kv7row[7]) 157 | row['DestinationCode'] = kv7row[8] 158 | row['TargetArrivalTime'] = kv7row[9] 159 | row['ExpectedArrivalTime'] = kv7row[9] 160 | row['TargetDepartureTime'] = kv7row[10] 161 | row['ExpectedDepartureTime'] = kv7row[10] 162 | row['SideCode'] = kv7row[11] 163 | if row['SideCode'] == '-': 164 | del(row['SideCode']) 165 | row['WheelChairAccessible'] = kv7row[12] 166 | row['JourneyStopType'] = kv7row[13] 167 | if not kv7row[14]: 168 | row['IsTimingStop'] = 0 169 | else: 170 | row['IsTimingStop'] = 1 171 | row['ProductFormulaType'] = kv7row[15] 172 | row['TimingPointCode'] = kv7row[16] 173 | row['TimingPointDataOwnerCode'] = kv7row[17] 174 | row['OperationDate'] = kv7row[18].strftime("%Y-%m-%d") 175 | row['JourneyPatternCode'] = kv7row[19] 176 | if row['JourneyPatternCode'] == None: 177 | row['JourneyPatternCode'] = 0 178 | row['TripStopStatus'] = 'PLANNED' 179 | row['LastUpdateTimeStamp'] = updatetimestamp 180 | pass_id = '_'.join([row['DataOwnerCode'], str(row['LocalServiceLevelCode']), row['LinePlanningNumber'], str(row['JourneyNumber']), str(row['FortifyOrderNumber']), row['UserStopCode'], str(row['UserStopOrderNumber'])]) 181 | passes['PASSTIMES'][pass_id] = row 182 | if (len(passes) > 50): 183 | push.send_json(passes) 184 | passes = {'PASSTIMES' : {} } 185 | cur.close() 186 | conn.close() 187 | push.send_json(passes) 188 | 189 | l = task.LoopingCall(fetchandpushkv7) 190 | l.start(60.0) # call every second 191 | reactor.run() 192 | -------------------------------------------------------------------------------- /kv7-db.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import zmq 3 | import simplejson as serializer 4 | import time 5 | from ctx import ctx 6 | from gzip import GzipFile 7 | from cStringIO import StringIO 8 | import psycopg2 9 | import time 10 | from datetime import datetime, timedelta 11 | from const import ZMQ_KV7 12 | from twisted.internet import task 13 | from twisted.internet import reactor 14 | 15 | conn = psycopg2.connect("dbname='kv78turbo' user='postgres' port='5433'") 16 | 17 | sys.stderr.write('Setting up a ZeroMQ PUSH: %s\n' % (ZMQ_KV7)) 18 | context = zmq.Context() 19 | push = context.socket(zmq.PUSH) 20 | push.connect(ZMQ_KV7) 21 | 22 | def secondsFromMidnight(time): 23 | hours, minutes, seconds = time.split(':') 24 | return (int(hours)*60*60) + (int(minutes)*60) + int(seconds) 25 | 26 | def time(seconds): 27 | hours = seconds / 3600 28 | if hours < 0: 29 | hours += 24 30 | seconds -= 3600*hours 31 | minutes = seconds / 60 32 | if minutes < 0: 33 | minutes *= -1 34 | seconds -= 60*minutes 35 | return "%02d:%02d:%02d" % (hours, minutes, seconds) 36 | 37 | now = datetime.now() + timedelta(hours=1) - timedelta(seconds=120) 38 | #now = datetime.now() + timedelta(minutes=30) - timedelta(seconds=120) 39 | 40 | def fetchandpushkv7(): 41 | passes = {} 42 | global now # this has to be done better 43 | now += timedelta(seconds=60) 44 | startrange = now.strftime("%H:%M:00") 45 | startdate = now.strftime("%Y-%m-%d") 46 | endrange = (datetime.now() + timedelta(hours=1)).strftime("%H:%M:00") 47 | #endrange = (datetime.now() + timedelta(minutes=30)).strftime("%H:%M:00") 48 | now = (datetime.now() + timedelta(hours=1) - timedelta(minutes=1)) 49 | #now = (datetime.now() + timedelta(minutes=30) - timedelta(minutes=1)) 50 | startdate48 = ((now + timedelta(seconds=60))-timedelta(days=1)).strftime("%Y-%m-%d") 51 | if endrange == '00:00:00': 52 | endrange = '24:00:00' 53 | shours,sminutes,sseconds = startrange.split(':') 54 | ehours,eminutes,eseconds = endrange.split(':') 55 | startrange48 = str(int(shours)+24) + ':' + sminutes + ':00' 56 | endrange48 = str(int(ehours)+24) + ':' + eminutes + ':00' 57 | sys.stdout.write(startrange + '-' + endrange + '@ ' + startdate) 58 | sys.stdout.write(' ') 59 | sys.stdout.write(startrange48 + '-' + endrange48 + '@ ' +startdate48 + '\n') 60 | cur = conn.cursor() 61 | cur.execute("select p.dataownercode,p.localservicelevelcode,p.lineplanningnumber,journeynumber,fortifyordernumber,p.userstopcode,userstopordernumber,linedirection, p.destinationcode,targetarrivaltime,targetdeparturetime,sidecode,wheelchairaccessible,journeystoptype,istimingstop,productformulatype,timingpointcode, timingpointdataownercode,operationdate from localservicegrouppasstime as p, usertimingpoint as u, localservicegroupvalidity as v where exists ( SELECT 1 FROM localservicegrouppasstime AS f, localservicegroupvalidity as v WHERE f.journeystoptype = 'FIRST' AND f.dataownercode = p.dataownercode AND f.localservicelevelcode = p.localservicelevelcode AND f.lineplanningnumber = p.lineplanningnumber and f.journeynumber = p.journeynumber AND f.fortifyordernumber = p.fortifyordernumber AND ((operationdate = date %s AND targetarrivaltime >= %s AND targetarrivaltime < %s ) OR (operationdate = date %s AND targetarrivaltime >= %s AND targetarrivaltime < %s)) AND f.localservicelevelcode = v.localservicelevelcode AND f.dataownercode = v.dataownercode) AND p.dataownercode = u.dataownercode AND p.userstopcode = u.userstopcode AND journeystoptype != 'INFOPOINT' AND p.localservicelevelcode = v.localservicelevelcode AND p.dataownercode = v.dataownercode AND ((operationdate = date %s AND targetarrivaltime >= %s and targetarrivaltime < %s) OR (operationdate = date %s AND targetarrivaltime >= %s));", [startdate, startrange,endrange,startdate48,startrange48,endrange48,startdate,startrange,startrange48,startdate48,startrange48]) 62 | kv7rows = cur.fetchall() 63 | passes = {} 64 | updatetimestamp = datetime.today().strftime("%Y-%m-%dT%H:%M:%S") + "+02:00" 65 | print str(len(kv7rows)) + ' rows from db' 66 | for kv7row in kv7rows: 67 | row = {} 68 | row['DataOwnerCode'] = kv7row[0] 69 | row['LocalServiceLevelCode'] = kv7row[1] 70 | row['LinePlanningNumber'] = kv7row[2] 71 | row['JourneyNumber'] = kv7row[3] 72 | row['FortifyOrderNumber'] = kv7row[4] 73 | row['UserStopCode'] = kv7row[5] 74 | row['UserStopOrderNumber'] = kv7row[6] 75 | row['LineDirection'] = int(kv7row[7]) 76 | row['DestinationCode'] = kv7row[8] 77 | row['TargetArrivalTime'] = kv7row[9] 78 | row['ExpectedArrivalTime'] = kv7row[9] 79 | row['TargetDepartureTime'] = kv7row[10] 80 | row['ExpectedDepartureTime'] = kv7row[10] 81 | row['SideCode'] = kv7row[11] 82 | if row['SideCode'] == '-': 83 | del(row['SideCode']) 84 | row['WheelChairAccessible'] = kv7row[12] 85 | row['JourneyStopType'] = kv7row[13] 86 | row['IsTimingStop'] = kv7row[14] 87 | row['ProductFormulaType'] = kv7row[15] 88 | row['TimingPointCode'] = kv7row[16] 89 | row['OperationDate'] = kv7row[18].strftime("%Y-%m-%d") 90 | row['TripStopStatus'] = 'PLANNED' 91 | row['LastUpdateTimeStamp'] = updatetimestamp 92 | pass_id = '_'.join([row['DataOwnerCode'], str(row['LocalServiceLevelCode']), row['LinePlanningNumber'], str(row['JourneyNumber']), str(row['FortifyOrderNumber']), row['UserStopCode'], str(row['UserStopOrderNumber'])]) 93 | passes[pass_id] = row 94 | if (len(passes) > 50): 95 | push.send_json(passes) 96 | passes = {} 97 | cur.close() 98 | push.send_json(passes) 99 | 100 | l = task.LoopingCall(fetchandpushkv7) 101 | l.start(60.0) # call every second 102 | reactor.run() 103 | -------------------------------------------------------------------------------- /kv78turbo-api.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import time 3 | import zmq 4 | from const import ZMQ_KV8, ZMQ_KV78UWSGI, ZMQ_KV7 5 | from datetime import datetime, timedelta 6 | from time import strftime, gmtime 7 | from gzip import GzipFile 8 | from cStringIO import StringIO 9 | import psycopg2 10 | from copy import deepcopy 11 | import codecs 12 | from threading import Thread 13 | import simplejson as json 14 | 15 | output = codecs.open('/var/ovapi/kv7.openov.nl/GOVI/CURRENTDB', 'r', 'UTF-8') 16 | dbname = output.read().split('\n')[0] 17 | conn = psycopg2.connect("dbname='%s'" % (dbname)) 18 | 19 | tpc_store = {} 20 | stopareacode_store = {} 21 | line_store = {} 22 | journey_store = {} 23 | last_updatestore = {'DataOwner' : {}, 'Subscription' : {}} 24 | generalmessagestore = {} 25 | 26 | tpc_meta = {} 27 | line_meta = {} 28 | destination_meta = {} 29 | 30 | journeystoptypefiltered = False #Indicates whether journeystoptype was filtered during import KV7import 31 | 32 | print 'Start loading kv7 data' 33 | cur = conn.cursor() 34 | cur.execute("SELECT dataownercode,lineplanningnumber,linepublicnumber,linename,transporttype from line", []) 35 | rows = cur.fetchall() 36 | for row in rows: 37 | line_id = intern(row[0] + '_' + row[1]) 38 | line_meta[line_id] = {'LinePublicNumber' : intern(row[2]), 'LineName' : intern(row[3]), 'TransportType' : intern(row[4])} 39 | cur.close() 40 | 41 | cur = conn.cursor() 42 | cur.execute("SELECT dataownercode,destinationcode,destinationname50 from destination", []) 43 | rows = cur.fetchall() 44 | for row in rows: 45 | destination_id = intern(row[0] + '_' + row[1]) 46 | destination_meta[destination_id] = intern(row[2]) 47 | cur.close() 48 | 49 | cur = conn.cursor() 50 | if journeystoptypefiltered: 51 | cur.execute(""" 52 | SELECT timingpointcode,timingpointname,timingpointtown,stopareacode,ST_Y(the_geom)::NUMERIC(9,7) AS lat,ST_X(the_geom)::NUMERIC(8,7) AS lon 53 | FROM (select *,ST_Transform(st_setsrid(st_makepoint(coalesce(locationx_ew,0), coalesce(locationy_ns,0)), 28992), 4326) AS the_geom FROM timingpoint) as t 54 | """,[]) 55 | else: 56 | cur.execute(""" 57 | SELECT timingpointcode,timingpointname,timingpointtown,stopareacode,ST_Y(the_geom)::NUMERIC(9,7) AS lat,ST_X(the_geom)::NUMERIC(8,7) AS lon 58 | FROM 59 | (SELECT DISTINCT dataownercode,userstopcode FROM localservicegrouppasstime WHERE journeystoptype != 'INFOPOINT') as u 60 | JOIN usertimingpoint as ut USING (dataownercode,userstopcode) 61 | JOIN (select *,ST_Transform(st_setsrid(st_makepoint(coalesce(locationx_ew,0), coalesce(locationy_ns,0)), 28992), 4326) AS the_geom FROM timingpoint) as t USING (timingpointcode) 62 | """,[]) 63 | kv7rows = cur.fetchall() 64 | for kv7row in kv7rows: 65 | tpc_meta[intern(kv7row[0])] = {'TimingPointName' : intern(kv7row[1]), 'TimingPointTown' : intern(kv7row[2]), 'StopAreaCode' : kv7row[3], 'Latitude' : float(kv7row[4]), 'Longitude' : float(kv7row[5])} 66 | #if kv7row[6] != 'UNKNOWN' and kv7row[6] is not None: 67 | # tpc_meta[kv7row[0]]['TimingPointVisualAccessible'] = kv7row[6] 68 | #if kv7row[7] != 'UNKNOWN' and kv7row[7] is not None: 69 | # tpc_meta[kv7row[0]]['TimingPointWheelChairAccessible'] = kv7row[7] 70 | cur.close() 71 | 72 | cur = conn.cursor() 73 | cur.execute(""" 74 | SELECT DISTINCT on (p.dataownercode,p.lineplanningnumber,p.linedirection) 75 | p.dataownercode,p.lineplanningnumber,p.linedirection,p.destinationcode 76 | FROM ( 77 | SELECT distinct on (dataownercode,lineplanningnumber,linedirection) 78 | dataownercode,lineplanningnumber,linedirection,journeypatterncode from localservicegrouppasstime 79 | WHERE journeystoptype = 'LAST' 80 | ORDER BY dataownercode,lineplanningnumber,linedirection,userstopordernumber DESC) as ljp,localservicegrouppasstime as p 81 | WHERE 82 | p.dataownercode = ljp.dataownercode AND 83 | p.lineplanningnumber = ljp.lineplanningnumber AND 84 | p.linedirection = ljp.linedirection AND 85 | p.journeypatterncode = ljp.journeypatterncode AND 86 | p.journeystoptype = 'FIRST' 87 | """) 88 | kv7rows = cur.fetchall() 89 | for kv7row in kv7rows: 90 | line_id = '_'.join([kv7row[0],kv7row[1],str(kv7row[2])]) 91 | line_store[line_id] = {'Network': {}, 'Actuals': {}, 'Line' : {}} 92 | line_store[line_id]['Line'] = {'DataOwnerCode' : kv7row[0], 'LineDirection' : kv7row[2], 'LinePlanningNumber' : kv7row[1], 'DestinationCode' : kv7row[3]} 93 | if 'ARR_16001_1' in line_store: 94 | line_store['ARR_16001_1']['Line']['DestinationCode'] = '3000' 95 | cur.close() 96 | conn.close() 97 | print 'Loaded KV7 data' 98 | 99 | try: 100 | f = open('generalmessage.json','r') 101 | generalmessagestore = json.load(f) 102 | f.close() 103 | for id,msg in generalmessagestore.items(): 104 | if msg['TimingPointCode'] not in tpc_store: 105 | tpc_store[msg['TimingPointCode']] = {'Passes' : {}, 'GeneralMessages' : {}} 106 | tpc_store[msg['TimingPointCode']]['GeneralMessages'][id] = msg 107 | except Exception as e: 108 | print e 109 | 110 | def totimestamp(operationdate, timestamp, row): 111 | hours, minutes, seconds = timestamp.split(':') 112 | years, months, days = operationdate.split('-') 113 | if hours == 0 and minutes == 0 and seconds == 0: 114 | return int(0) 115 | hours = int(hours) 116 | if hours >= 48: 117 | print row 118 | if hours >= 24: 119 | deltadays = hours / 24 120 | hours = hours % 24 121 | return int((datetime(int(years), int(months), int(days), hours, int(minutes), int(seconds)) + timedelta(days = deltadays)).strftime("%s")) 122 | else: 123 | return int(datetime(int(years), int(months), int(days), hours, int(minutes), int(seconds)).strftime("%s")) 124 | 125 | def todate(timestamp): 126 | return datetime.fromtimestamp(timestamp).strftime("%Y-%m-%dT%H:%M:%S") 127 | 128 | def cleanup(): 129 | now = long((datetime.today() - timedelta(seconds=60)).strftime("%s")) 130 | for timingpointcode, values in tpc_store.items(): 131 | for journey, row in values['Passes'].items(): 132 | if now > row['ExpectedArrivalTime'] and now > row['ExpectedDepartureTime']: 133 | del(tpc_store[timingpointcode]['Passes'][journey]) 134 | if row['TripStopStatus'] in ['PLANNED','UNKNOWN','CANCEL']: 135 | userstoporder = row['UserStopOrderNumber'] 136 | if journey in journey_store and userstoporder in journey_store[journey]['Stops']: 137 | line_id = row['DataOwnerCode'] + '_' + row['LinePlanningNumber'] + '_' + str(row['LineDirection']) 138 | if line_id in line_store and journey in line_store[line_id]['Actuals']: 139 | del(line_store[line_id]['Actuals'][journey]) 140 | if len(journey_store[journey]['Stops']) > 1: 141 | del(journey_store[journey]['Stops'][userstoporder]) 142 | else: 143 | del(journey_store[journey]) 144 | for journey_id, values in journey_store.items(): 145 | row = values['Stops'][max(values['Stops'].keys())] 146 | if now > row['ExpectedArrivalTime'] and now > row['ExpectedDepartureTime']: 147 | line_id = row['DataOwnerCode'] + '_' + row['LinePlanningNumber'] + '_' + str(row['LineDirection']) 148 | if line_id in line_store and journey_id in line_store[line_id]['Actuals']: 149 | del(line_store[line_id]['Actuals'][journey_id]) 150 | if journey_id in journey_store: 151 | del(journey_store[journey_id]) 152 | for id,row in generalmessagestore.items(): 153 | if 'MessageEndTime' in row and now > row['MessageEndTime']: 154 | if row['TimingPointCode'] in tpc_store and id in tpc_store[row['TimingPointCode']]['GeneralMessages']: 155 | del(tpc_store[row['TimingPointCode']]['GeneralMessages'][id]) 156 | del(generalmessagestore[id]) 157 | 158 | def setDelayIncrease(oldrow,newrow): 159 | if newrow['TripStopStatus'] == 'DRIVING' and oldrow['TripStopStatus'] != 'PLANNED' and oldrow['JourneyStopType'] != 'LAST': 160 | targetdeparture = oldrow['TargetDepartureTime'] 161 | timediff = (newrow['LastUpdateTimeStamp'] - oldrow['LastUpdateTimeStamp']) 162 | olddelay = (oldrow['ExpectedDepartureTime'] - targetdeparture) 163 | newdelay = (newrow['ExpectedDepartureTime'] - targetdeparture) 164 | delaydiff = (newdelay - olddelay) 165 | if delaydiff > timediff and newdelay > 60: 166 | oldrow['JourneyDisrupted'] = True 167 | return 168 | if 'JourneyDisrupted' in oldrow: 169 | del(oldrow['JourneyDisrupted']) 170 | 171 | def storecurrect(newrow): 172 | newrow['ExpectedArrivalTime'] = totimestamp(newrow['OperationDate'], newrow['ExpectedArrivalTime'], newrow) 173 | newrow['ExpectedDepartureTime'] = totimestamp(newrow['OperationDate'], newrow['ExpectedDepartureTime'], newrow) 174 | newrow['TargetArrivalTime'] = totimestamp(newrow['OperationDate'], newrow['TargetArrivalTime'], newrow) 175 | newrow['TargetDepartureTime'] = totimestamp(newrow['OperationDate'], newrow['TargetDepartureTime'], newrow) 176 | date,time = newrow['LastUpdateTimeStamp'].split('T') 177 | newrow['LastUpdateTimeStamp'] = totimestamp(date,time[:-6],None) 178 | 179 | id = '_'.join([newrow['DataOwnerCode'], str(newrow['LocalServiceLevelCode']), newrow['LinePlanningNumber'], str(newrow['JourneyNumber']), str(newrow['FortifyOrderNumber'])]) 180 | if id in journey_store and int(newrow['UserStopOrderNumber']) in journey_store[id]['Stops']: 181 | row = journey_store[id]['Stops'][int(newrow['UserStopOrderNumber'])] 182 | if row['TripStopStatus'] == 'LINESTOPCANCEL': 183 | newrow['TripStopStatus'] = 'LINESTOPCANCEL' 184 | setDelayIncrease(row,newrow) 185 | if newrow['WheelChairAccessible'] == 'UNKNOWN': 186 | newrow['WheelChairAccessible'] = row['WheelChairAccessible'] # Because of agencies not implementing the accessiblity tag in KV6, we're better off ignoring UKNOWNS unfortunately 187 | row.update(newrow) 188 | else: 189 | row = newrow 190 | 191 | line_id = row['DataOwnerCode'] + '_' + row['LinePlanningNumber'] + '_' + str(row['LineDirection']) 192 | linemeta_id = row['DataOwnerCode'] + '_' + row['LinePlanningNumber'] 193 | destinationmeta_id = row['DataOwnerCode'] + '_' + row['DestinationCode'] 194 | pass_id = '_'.join([row['UserStopCode'], str(row['UserStopOrderNumber'])]) 195 | 196 | for x in ['JourneyPatternCode', 'JourneyNumber', 'FortifyOrderNumber', 'UserStopOrderNumber', 'NumberOfCoaches', 'LocalServiceLevelCode', 'LineDirection']: 197 | try: 198 | if x in row and row[x] is not None and row[x] != 'UNKNOWN': 199 | row[x] = int(row[x]) 200 | else: 201 | del(row[x]) 202 | except: 203 | pass 204 | row['IsTimingStop'] = (row['IsTimingStop'] == '1') 205 | 206 | if row['TimingPointCode'] not in tpc_store: 207 | tpc_store[row['TimingPointCode']] = {'Passes' : {id: row}, 'GeneralMessages' : {}} 208 | else: 209 | tpc_store[row['TimingPointCode']]['Passes'][id] = row 210 | 211 | if row['TimingPointCode'] in tpc_meta: 212 | stopareacode = tpc_meta[row['TimingPointCode']]['StopAreaCode'] 213 | if stopareacode != None: 214 | if stopareacode not in stopareacode_store: 215 | stopareacode_store[stopareacode] = [row['TimingPointCode']] 216 | elif row['TimingPointCode'] not in stopareacode_store[stopareacode]: 217 | stopareacode_store[stopareacode].append(row['TimingPointCode']) 218 | 219 | if line_id not in line_store: 220 | line_store[line_id] = {'Network': {}, 'Actuals': {}, 'Line' : {}} 221 | line_store[line_id]['Line'] = {'DataOwnerCode' : row['DataOwnerCode'], 'LineDirection' : row['LineDirection'], 'LinePlanningNumber' : row['LinePlanningNumber'], 'DestinationCode' : row['DestinationCode']} 222 | if row['JourneyPatternCode'] not in line_store[line_id]['Network']: 223 | line_store[line_id]['Network'][row['JourneyPatternCode']] = {} 224 | if row['UserStopOrderNumber'] not in line_store[line_id]['Network'][row['JourneyPatternCode']]: 225 | line_store[line_id]['Network'][row['JourneyPatternCode']][row['UserStopOrderNumber']] = { 226 | 'TimingPointCode': row['TimingPointCode'], 227 | 'IsTimingStop': row['IsTimingStop'], 228 | 'UserStopOrderNumber':row['UserStopOrderNumber'] 229 | } 230 | if id not in journey_store: 231 | journey_store[id] = {'Stops' : {row['UserStopOrderNumber']: row}} 232 | if row['WheelChairAccessible'] != 'UNKNOWN': 233 | line_store[line_id]['Line']['LineWheelchairAccessible'] = row['WheelChairAccessible'] 234 | else: 235 | journey_store[id]['Stops'][row['UserStopOrderNumber']] = row 236 | 237 | if row['TripStopStatus'] in set(['ARRIVED', 'PASSED']): # , 'DRIVING']): Driving alleen nemen als kleinste waarde uit lijn, gegeven dat er geen ARRIVED/PASSED is 238 | for key in journey_store[id]['Stops'].keys(): #delete previous stops from journey 239 | if key < int(row['UserStopOrderNumber']) - 1: 240 | del(journey_store[id]['Stops'][key]) 241 | 242 | if row['JourneyStopType'] == 'LAST': #delete journey 243 | if id in line_store[line_id]['Actuals']: 244 | del(line_store[line_id]['Actuals'][id]) 245 | else: 246 | line_store[line_id]['Actuals'][id] = row 247 | elif row['TripStopStatus'] == 'DRIVING': #replace a passed stop with the next stop 248 | previousStopOrder = int(row['UserStopOrderNumber']) - 1 249 | if previousStopOrder in journey_store[id]['Stops'] and journey_store[id]['Stops'][previousStopOrder]['TripStopStatus'] == 'PASSED': 250 | line_store[line_id]['Actuals'][id] = row 251 | elif row['TripStopStatus'] == 'PLANNED' and row['TimingPointDataOwnerCode'] == 'ALGEMEEN' and id not in line_store[line_id]['Actuals'] and int(row['UserStopOrderNumber']) == 1: #add planned journeys 252 | line_store[line_id]['Actuals'][id] = row 253 | elif (row['TripStopStatus'] == 'UNKNOWN' or row['TripStopStatus'] == 'CANCEL') and id in line_store[line_id]['Actuals']: #Delete canceled or non live journeys 254 | del(line_store[line_id]['Actuals'][id]) 255 | 256 | def savemsgstore(): 257 | try: 258 | f = open('generalmessage.json','w') 259 | json.dump(generalmessagestore,f) 260 | f.close() 261 | except: 262 | pass 263 | 264 | def storemessage(row): 265 | id = '_'.join([row['DataOwnerCode'], row['MessageCodeDate'], row['MessageCodeNumber'], row['TimingPointDataOwnerCode'], row['TimingPointCode']]) 266 | if row['MessageEndTime'] is None or int(row['MessageEndTime'][0:4]) < 1900: 267 | return 268 | row['MessageEndTime'] = int(datetime.strptime(row['MessageEndTime'][:-6],"%Y-%m-%dT%H:%M:%S").strftime("%s")) 269 | row['MessageStartTime'] = int(datetime.strptime(row['MessageStartTime'][:-6],"%Y-%m-%dT%H:%M:%S").strftime("%s")) 270 | row['MessageTimeStamp'] = int(datetime.strptime(row['MessageTimeStamp'][:-6],"%Y-%m-%dT%H:%M:%S").strftime("%s")) 271 | if row['TimingPointCode'] in tpc_store: 272 | tpc_store[row['TimingPointCode']]['GeneralMessages'][id] = row 273 | else: 274 | tpc_store[row['TimingPointCode']] = {'Passes' : {}, 'GeneralMessages' : {id : row}} 275 | generalmessagestore[id] = row 276 | savemsgstore() 277 | 278 | def deletemessage(row): 279 | id = '_'.join([row['DataOwnerCode'], row['MessageCodeDate'], row['MessageCodeNumber'], row['TimingPointDataOwnerCode'], row['TimingPointCode']]) 280 | if row['TimingPointCode'] in tpc_store and id in tpc_store[row['TimingPointCode']]['GeneralMessages']: 281 | del(tpc_store[row['TimingPointCode']]['GeneralMessages'][id]) 282 | if id in generalmessagestore: 283 | del(generalmessagestore[id]) 284 | savemsgstore() 285 | 286 | 287 | context = zmq.Context() 288 | 289 | kv8 = context.socket(zmq.SUB) 290 | kv8.connect(ZMQ_KV8) 291 | kv8.setsockopt(zmq.SUBSCRIBE, "/GOVI/KV8") 292 | 293 | kv7 = context.socket(zmq.PULL) 294 | kv7.bind(ZMQ_KV7) 295 | 296 | poller = zmq.Poller() 297 | poller.register(kv8, zmq.POLLIN) 298 | poller.register(kv7, zmq.POLLIN) 299 | 300 | garbage = 0 301 | 302 | def addMeta(passtimes,terminating): 303 | result = {} 304 | for key, values in passtimes.items(): 305 | if values['JourneyStopType'] != 'LAST' or terminating: 306 | result[key] = values.copy() 307 | linemeta_id = values['DataOwnerCode'] + '_' + values['LinePlanningNumber'] 308 | if linemeta_id in line_meta: 309 | result[key].update(line_meta[linemeta_id]) 310 | destinationmeta_id = values['DataOwnerCode'] + '_' + values['DestinationCode'] 311 | if destinationmeta_id in destination_meta: 312 | result[key]['DestinationName50'] = destination_meta[destinationmeta_id] 313 | timingpointcode = values['TimingPointCode'] 314 | if timingpointcode in tpc_meta: 315 | result[key].update(tpc_meta[timingpointcode]) 316 | result[key]['ExpectedDepartureTime'] = todate(result[key]['ExpectedDepartureTime']) 317 | result[key]['ExpectedArrivalTime'] = todate(result[key]['ExpectedArrivalTime']) 318 | result[key]['TargetDepartureTime'] = todate(result[key]['TargetDepartureTime']) 319 | result[key]['TargetArrivalTime'] = todate(result[key]['TargetArrivalTime']) 320 | if result[key]['TripStopStatus'] == 'LINESTOPCANCEL': 321 | result[key]['TripStopStatus'] = 'CANCEL' 322 | if 'LastUpdateTimeStamp' in values: 323 | result[key]['LastUpdateTimeStamp'] = todate(result[key]['LastUpdateTimeStamp']) 324 | return result 325 | 326 | def queryTimingPoints(arguments): 327 | if arguments[0] == 'tpc': 328 | if len(arguments) == 1: 329 | reply = {} 330 | for tpc, values in tpc_store.items(): 331 | reply[tpc] = len(values['Passes']) 332 | return reply 333 | else: 334 | reply = {} 335 | for tpc in set(arguments[1].split(',')): 336 | if tpc in tpc_store and tpc != '': 337 | reply[tpc] = tpc_store[tpc].copy() 338 | now = long((datetime.today()).strftime("%s")) 339 | reply[tpc]['GeneralMessages'] = deepcopy(tpc_store[tpc]['GeneralMessages']) 340 | for key,value in reply[tpc]['GeneralMessages'].items(): 341 | if value['MessageStartTime'] > now: 342 | del(reply[tpc]['GeneralMessages'][key]) 343 | else: 344 | if 'MessageEndTime' in value: 345 | value['MessageEndTime'] = todate(value['MessageEndTime']) 346 | value['MessageStartTime'] = todate(value['MessageStartTime']) 347 | value['MessageTimeStamp'] = todate(value['MessageTimeStamp']) 348 | reply[tpc]['Passes'] = addMeta(reply[tpc]['Passes'],(arguments[-1] != 'departures')) 349 | if tpc in tpc_meta and tpc != '': 350 | if tpc in reply: 351 | reply[tpc]['Stop'] = tpc_meta[tpc] 352 | reply[tpc]['Stop']['TimingPointCode'] = tpc 353 | else: 354 | reply[tpc] = {'Stop' : tpc_meta[tpc], 'GeneralMessages' : {}, 'Passes' : {}} 355 | tpc_store[tpc] = {'Stop' : tpc_meta[tpc], 'GeneralMessages' : {}, 'Passes' : {}} 356 | reply[tpc]['Stop']['TimingPointCode'] = tpc 357 | return reply 358 | 359 | def queryJourneys(arguments): 360 | if len(arguments) == 1: 361 | reply = {} 362 | for journey, values in journey_store.items(): 363 | reply[journey] = len(values['Stops']) 364 | return reply 365 | else: 366 | reply = {} 367 | for journey in set(arguments[1].split(',')): 368 | if journey in journey_store: 369 | if journey != '': 370 | reply[journey] = journey_store[journey].copy() 371 | reply[journey]['ServerTime'] = strftime("%Y-%m-%dT%H:%M:%SZ",gmtime()) 372 | reply[journey]['Stops'] = addMeta(reply[journey]['Stops'],True) 373 | return reply 374 | 375 | def queryStopAreas(arguments): 376 | if len(arguments) == 1: 377 | reply = {} 378 | for stopareacode in stopareacode_store: 379 | for tpc in stopareacode_store[stopareacode]: 380 | if tpc in tpc_meta: 381 | reply[stopareacode] = tpc_meta[tpc].copy() 382 | if 'TimingPointWheelChairAccessible' in reply[stopareacode]: 383 | del(reply[stopareacode]['TimingPointWheelChairAccessible']) 384 | if 'TimingPointVisualAccessible' in reply[stopareacode]: 385 | del(reply[stopareacode]['TimingPointVisualAccessible']) 386 | return reply 387 | else: 388 | reply = {} 389 | for stopareacode in set(arguments[1].split(',')): 390 | if stopareacode in stopareacode_store and stopareacode != '': 391 | reply[stopareacode] = {} 392 | reply[stopareacode]['ServerTime'] = strftime("%Y-%m-%dT%H:%M:%SZ",gmtime()) 393 | for tpc in stopareacode_store[stopareacode]: 394 | if tpc in tpc_store and tpc != '': 395 | reply[stopareacode][tpc] = tpc_store[tpc].copy() 396 | reply[stopareacode][tpc]['GeneralMessages'] = deepcopy(tpc_store[tpc]['GeneralMessages']) 397 | now = long((datetime.today()).strftime("%s")) 398 | for key,value in reply[stopareacode][tpc]['GeneralMessages'].items(): 399 | if value['MessageStartTime'] > now: 400 | del(reply[stopareacode][tpc]['GeneralMessages'][key]) 401 | else: 402 | if 'MessageEndTime' in value: 403 | value['MessageEndTime'] = todate(value['MessageEndTime']) 404 | value['MessageStartTime'] = todate(value['MessageStartTime']) 405 | value['MessageTimeStamp'] = todate(value['MessageTimeStamp']) 406 | reply[stopareacode][tpc]['Passes'] = addMeta(reply[stopareacode][tpc]['Passes'],(arguments[-1] != 'departures')) 407 | if tpc in tpc_meta: 408 | if tpc in reply: 409 | reply[stopareacode][tpc]['Stop'] = tpc_meta[tpc] 410 | else: 411 | reply[stopareacode][tpc] = {'Stop' : tpc_meta[tpc], 'GeneralMessages' : {}, 'Passes' : {}} 412 | tpc_store[tpc] = {'Stop' : tpc_meta[tpc], 'GeneralMessages' : {}, 'Passes' : {}} 413 | return reply 414 | 415 | def queryLines(arguments,no_network=False): 416 | if len(arguments) == 1: 417 | reply = {} 418 | for line, values in line_store.items(): 419 | if len(values['Network']) > 0: 420 | reply[line] = values['Line'].copy() 421 | linemeta_id = values['Line']['DataOwnerCode'] + '_' + values['Line']['LinePlanningNumber'] 422 | if 'DestinationCode' in values['Line']: 423 | destination_id = values['Line']['DataOwnerCode']+'_'+values['Line']['DestinationCode'] 424 | if destination_id in destination_meta: 425 | reply[line]['DestinationName50'] = destination_meta[destination_id] 426 | if linemeta_id in line_meta: 427 | reply[line].update(line_meta[linemeta_id]) 428 | return reply 429 | else: 430 | reply = {} 431 | for line in set(arguments[1].split(',')): 432 | if line in line_store and line != '': 433 | if len(line_store[line]['Network']) == 0: 434 | continue 435 | reply[line] = deepcopy(line_store[line]) 436 | reply[line]['ServerTime'] = strftime("%Y-%m-%dT%H:%M:%SZ",gmtime()) 437 | reply[line]['Actuals'] = addMeta(reply[line]['Actuals'],True) 438 | line_id = reply[line]['Line']['DataOwnerCode']+'_'+reply[line]['Line']['LinePlanningNumber'] 439 | if line_id in line_meta: 440 | reply[line]['Line'].update(line_meta[line_id]) 441 | if 'DestinationCode' in reply[line]['Line']: 442 | destination_id = reply[line]['Line']['DataOwnerCode']+'_'+reply[line]['Line']['DestinationCode'] 443 | if destination_id in destination_meta: 444 | reply[line]['Line']['DestinationName50'] = destination_meta[destination_id] 445 | if no_network: 446 | del(reply[line]['Network']) 447 | else: 448 | for journeypatterncode,journeypattern in reply[line]['Network'].items(): 449 | for userstoporder, timingpoint in journeypattern.items(): 450 | if timingpoint['TimingPointCode'] in tpc_meta: 451 | timingpoint.update(tpc_meta[timingpoint['TimingPointCode']]) 452 | return reply 453 | 454 | def recvPackage(content): 455 | for line in content.split('\r\n')[:-1]: 456 | if line[0] == '\\': 457 | # control characters 458 | if line[1] == 'G': 459 | label, name, subscription, path, endian, enc, res1, timestamp, _ = line[2:].split('|') 460 | elif line[1] == 'T': 461 | type = line[2:].split('|')[1] 462 | elif line[1] == 'L': 463 | keys = line[2:].split('|') 464 | else: 465 | row = {} 466 | values = line.split('|') 467 | for k,v in map(None, keys, values): 468 | if v == '\\0': 469 | row[k] = None 470 | else: 471 | row[k] = v 472 | for x in ['ReasonType', 'AdviceType', 'AdviceContent','SubAdviceType','MessageType','ReasonContent','OperatorCode', 'SubReasonType', 'MessageContent']: 473 | if x in row and row[x] is None: 474 | del(row[x]) 475 | if type == 'DATEDPASSTIME': 476 | if 'SideCode' in row and row['SideCode'] in ['-','Left','Right']: 477 | del(row['SideCode']) 478 | elif 'SideCode' in row: 479 | row['SideCode'] = intern(row['SideCode']) 480 | if row['TripStopStatus'] != 'UNKNOWN' and row['TripStopStatus'] != 'PLANNED': #Keeps status of the dataowners supplying us data 481 | last_updatestore['DataOwner'][row['DataOwnerCode']] = row['LastUpdateTimeStamp'] 482 | last_updatestore['Subscription'][subscription] = row['LastUpdateTimeStamp'] 483 | elif row['DataOwnerCode'] not in last_updatestore['DataOwner']: 484 | last_updatestore['DataOwner'][row['DataOwnerCode']] = 'ERROR' 485 | if subscription not in last_updatestore['Subscription']: 486 | last_updatestore['Subscription'][subscription] = 'ERROR' 487 | if row['JourneyStopType'] != 'INFOPOINT': 488 | storecurrect(row) 489 | elif type == 'GENERALMESSAGEUPDATE': 490 | print 'GENERAL MESSAGE UPDATE' 491 | storemessage(row) 492 | print content 493 | elif type == 'GENERALMESSAGEDELETE': 494 | print 'GENERAL MESSAGE DELETE' 495 | deletemessage(row) 496 | print content 497 | else: 498 | print 'UNKNOWN TYPE : !!!!!' + type 499 | print content 500 | 501 | class read(Thread): 502 | def __init__ (self): 503 | Thread.__init__(self) 504 | def run(self): 505 | client = context.socket(zmq.REP) 506 | client.bind(ZMQ_KV78UWSGI) 507 | while True: 508 | url = client.recv() 509 | try: 510 | arguments = url.split('/') 511 | if arguments[0] == 'tpc': 512 | reply = queryTimingPoints(arguments) 513 | client.send_json(reply) 514 | elif arguments[0] == 'journey': 515 | reply = queryJourneys(arguments) 516 | client.send_json(reply) 517 | elif arguments[0] == 'stopareacode': 518 | reply = queryStopAreas(arguments) 519 | client.send_json(reply) 520 | elif arguments[0] == 'line': 521 | reply = queryLines(arguments,no_network=(arguments[-1] == 'actuals')) 522 | client.send_json(reply) 523 | elif arguments[0] == 'lastupdate': 524 | reply = {'LastUpdateTimeStamps' : last_updatestore, 'ServerTime' : strftime("%Y-%m-%dT%H:%M:%SZ",gmtime())} 525 | client.send_json(reply) 526 | elif arguments[0] == 'generalmessage': 527 | client.send_json(generalmessagestore) 528 | elif arguments[0] == 'admin': 529 | client.send_json(['YO']) 530 | else: 531 | client.send_json([]) 532 | except Exception as e: 533 | client.send_json([]) 534 | print e 535 | Thread.__init__(self) 536 | 537 | 538 | thread = read() 539 | thread.start() 540 | 541 | while True: 542 | socks = dict(poller.poll()) 543 | if socks.get(kv8) == zmq.POLLIN: 544 | multipart = kv8.recv_multipart() 545 | content = GzipFile('','r',0,StringIO(''.join(multipart[1:]))).read() 546 | recvPackage(content) 547 | elif socks.get(kv7) == zmq.POLLIN: 548 | data = kv7.recv_json() 549 | try: 550 | if 'PASSTIMES' in data: 551 | for pass_id, row in data['PASSTIMES'].items(): 552 | id = '_'.join([row['DataOwnerCode'], str(row['LocalServiceLevelCode']), row['LinePlanningNumber'], str(row['JourneyNumber']), str(row['FortifyOrderNumber'])]) 553 | if id not in journey_store or int(row['UserStopOrderNumber']) not in journey_store[id]['Stops']: 554 | try: 555 | storecurrect(row) 556 | except Exception as e: 557 | print e 558 | if 'DESTINATION' in data: 559 | for dest_id, dest in data['DESTINATION'].items(): 560 | destination_meta[dest_id] = dest 561 | if 'TIMINGPOINT' in data: 562 | for tpc, timingpoint in data['TIMINGPOINT'].items(): 563 | tpc_meta[tpc] = timingpoint 564 | if 'LINE' in data: 565 | for line_id, line in data['LINE'].items(): 566 | line_meta[line_id] = line 567 | if 'NETWORK' in data: 568 | for line_id,network in data['NETWORK'].items(): 569 | newnetwork = {} 570 | for jpcode,jp in network.items(): 571 | if jpcode not in newnetwork: 572 | newnetwork[jpcode] = {} 573 | for stoporder,stop in jp.items(): 574 | newnetwork[jpcode][int(stoporder)] = stop 575 | if line_id not in line_store: 576 | line_store[line_id] = {'Network': {}, 'Actuals': {}, 'Line' : {}} 577 | line_store[line_id]['Network'] = newnetwork 578 | if 'LINEMETA' in data: 579 | for line_id, meta in data['LINEMETA'].items(): 580 | if line_id in line_store: 581 | line_store[line_id]['Line']['DataOwnerCode'] = meta['DataOwnerCode'] 582 | line_store[line_id]['Line']['LineDirection'] = meta['LineDirection'] 583 | line_store[line_id]['Line']['LinePlanningNumber'] = meta['LinePlanningNumber'] 584 | line_store[line_id]['Line']['DestinationCode'] = meta['DestinationCode'] 585 | except Exception as e: 586 | print e 587 | if garbage > 200: 588 | cleanup() 589 | garbage = 0 590 | else: 591 | garbage += 1 592 | -------------------------------------------------------------------------------- /kv78turbo-client.py: -------------------------------------------------------------------------------- 1 | import uwsgi 2 | import zmq 3 | from const import ZMQ_KV78UWSGI 4 | import simplejson 5 | import time 6 | 7 | COMMON_HEADERS = [('Content-Type', 'application/json'), ('Access-Control-Allow-Origin', '*'), ('Access-Control-Allow-Headers', 'Requested-With,Content-Type')] 8 | 9 | lines = {} 10 | stopareas = {} 11 | 12 | def notfound(start_response): 13 | start_response('404 File Not Found', COMMON_HEADERS + [('Content-length', '2')]) 14 | yield '[]' 15 | 16 | def KV78Client(environ, start_response): 17 | url = environ['PATH_INFO'][1:] 18 | if len(url) > 0 and url[-1] == '/': 19 | url = url[:-1] 20 | 21 | arguments = url.split('/') 22 | if arguments[0] not in set(['tpc','journey','line','linesgh','stopareacode','lastupdate','generalmessage','line2']) or len(arguments) > 3: 23 | return notfound(start_response) 24 | 25 | context = zmq.Context() 26 | client = context.socket(zmq.REQ) 27 | client.connect(ZMQ_KV78UWSGI) 28 | if arguments[0] == 'line2': 29 | if 'data2' not in lines or lines['time2'] < (time.time() - 1000): 30 | client.send('line') 31 | data = {} 32 | for line_id,line in client.recv_json().items(): 33 | if 'DestinationName50' in line: 34 | del(line['DestinationName50']) 35 | if 'LineDirection' in line: 36 | del(line['LineDirection']) 37 | if 'DestinationCode' in line: 38 | del(line['DestinationCode']) 39 | data[line_id[:-2]] = line 40 | lines['data2'] = simplejson.dumps(data) 41 | lines['time2'] = time.time() 42 | reply = lines['data2'] 43 | elif arguments[0] == 'linesgh': 44 | if 'sgh' not in lines or lines['sghtime'] < time.time() - 1000: 45 | client.send('line') 46 | data = client.recv_json() 47 | sghlines = {} 48 | for key,value in data.items(): 49 | if key.startswith('HTM'): 50 | sghlines[key] = value 51 | elif key.startswith('CXX_R'): 52 | sghlines[key] = value 53 | elif key.startswith('VTN_40'): 54 | sghlines[key] = value 55 | elif key.startswith('CXX_W'): 56 | sghlines[key] = value 57 | elif key.startswith('QBUZZ_r270') or key.startswith('QBUZZ_r170'): 58 | sghlines[key] = value 59 | lines['sgh'] = simplejson.dumps(sghlines) 60 | lines['sghtime'] = time.time() 61 | reply = lines['sgh'] 62 | elif len(arguments) == 1 and arguments[0] == 'line': 63 | if 'data' not in lines or lines['time'] < (time.time() - 1000): 64 | client.send(url) 65 | lines['data'] = client.recv() 66 | lines['time'] = time.time() 67 | print 'Line cache miss' 68 | reply = lines['data'] 69 | elif (len(arguments) == 1 and arguments[0] == 'stopareacode'): 70 | if 'data' not in stopareas or stopareas['time'] < time.time() - 1000: 71 | client.send(url) 72 | stopareas['data'] = client.recv() 73 | stopareas['time'] = time.time() 74 | reply = stopareas['data'] 75 | else: 76 | client.send(url) 77 | reply = client.recv() 78 | if len(reply) < 3: 79 | return notfound(start_response) 80 | 81 | start_response('200 OK', COMMON_HEADERS + [('Content-length', str(len(reply)))]) 82 | return reply 83 | 84 | uwsgi.applications = {'': KV78Client} 85 | -------------------------------------------------------------------------------- /nsavt.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import zmq 3 | import simplejson as serializer 4 | from datetime import datetime, timedelta 5 | from time import strftime, strptime, gmtime 6 | from gzip import GzipFile 7 | from ctx import ctx 8 | from gzip import GzipFile 9 | from cStringIO import StringIO 10 | import calendar 11 | 12 | ZMQ_PUBSUB_NS = "tcp://83.98.158.170:6611" 13 | ZMQ_KV7 = "tcp://127.0.0.1:6070" 14 | 15 | # Initialize a zeromq CONTEXT 16 | context = zmq.Context() 17 | sys.stderr.write('Setting up a ZeroMQ SUB: %s\n' % (ZMQ_PUBSUB_NS)) 18 | subscribe_kv8 = context.socket(zmq.SUB) 19 | subscribe_kv8.connect(ZMQ_PUBSUB_NS) 20 | subscribe_kv8.setsockopt(zmq.SUBSCRIBE, '') 21 | 22 | sys.stderr.write('Setting up a ZeroMQ PUSH: %s\n' % (ZMQ_KV7)) 23 | context = zmq.Context() 24 | push = context.socket(zmq.PUSH) 25 | push.connect(ZMQ_KV7) 26 | 27 | # Set up a poller 28 | poller = zmq.Poller() 29 | poller.register(subscribe_kv8, zmq.POLLIN) 30 | 31 | # Cache 32 | actuals = {} 33 | 34 | def productformula(treinsoort): 35 | if treinsoort == 'Stoptrein': 36 | return 25 37 | if treinsoort == 'Sneltrein': 38 | return 26 39 | if treinsoort == 'Intercity': 40 | return 27 41 | if treinsoort == 'Sprinter': 42 | return 28 43 | if treinsoort == 'Internationale trein': 44 | return 29 45 | if treinsoort == 'Fyra': 46 | return 30 47 | if treinsoort == 'ICE International': 48 | return 31 49 | if treinsoort == 'Thalys': 50 | return 32 51 | if treinsoort == 'CityNightLine': 52 | return 29 53 | if treinsoort == 'Stopbus i.p.v. Trein': 54 | return 18 55 | print treinsoort 56 | return 0 57 | 58 | def dataownercode(vervoerder): 59 | if vervoerder == 'Veolia': 60 | return 'VTN' 61 | if vervoerder == 'Arriva': 62 | return 'ARR' 63 | if vervoerder == 'Connexxion': 64 | return 'CXX' 65 | if vervoerder == 'Syntus': 66 | return 'SYNTUS' 67 | return 'NS' 68 | 69 | def linenumber(ritnummer): 70 | try: 71 | journeynumber = int(ritnummer) 72 | except: 73 | pass 74 | return 0 75 | if journeynumber > 0 and journeynumber < 999: 76 | return 10 * (journeynumber / 10) 77 | if journeynumber > 999 and journeynumber < 9999: 78 | return 100 * (journeynumber / 100) 79 | if journeynumber > 9999 and journeynumber < 99999: 80 | return 1000 * (journeynumber / 1000) 81 | if journeynumber > 99999 and journeynumber < 999999: 82 | return 10000 * (journeynumber / 10000) 83 | if journeynumber > 999999 and journeynumber < 9999999: 84 | return 100000 * (journeynumber / 100000) 85 | 86 | def generateuserstopordernumber(targetdeparturetime): 87 | #as we dont have a slight, hack something up using the departuretme 88 | return calendar.timegm(targetdeparturetime.utctimetuple()) % 10000 89 | 90 | def convertkv(avt): 91 | row = {} 92 | try: 93 | row['JourneyNumber'] = avt['RitNummer'] 94 | except: 95 | row['JourneyNumber'] = 0 96 | row['DestinationName50'] = avt['EindBestemming'] 97 | row['DestinationCode'] = row['DestinationName50'] 98 | row['TripStopStatus'] = 'DRIVING' 99 | if avt['Opmerkingen'] == 'Trein rijdt niet': 100 | row['TripStopStatus'] = 'CANCEL' 101 | else: 102 | row['MessageContent'] = avt['Opmerkingen'] 103 | row['MessageContent'] = avt['Opmerkingen'] 104 | row['AdviceContent'] = avt['ReisTip'] 105 | row['SideCode'] = avt['VertrekSpoor'] 106 | row['LastUpdateTimeStamp'] = datetime.today().strftime("%Y-%m-%dT%H:%M:%S") 107 | row['ProductFormulaType'] = productformula(avt['TreinSoort']) 108 | row['DataOwnerCode'] = dataownercode(avt['Vervoerder']) 109 | targetdeparturetime = datetime.strptime(avt['VertrekTijd'].split('+')[0], "%Y-%m-%d %H:%M:%S") 110 | expecteddeparturetime = targetdeparturetime + timedelta(minutes=int(avt['VertrekVertraging'])) 111 | row['TargetDepartureTime'] = targetdeparturetime.strftime("%H:%M:%S") 112 | row['ExpectedDepartureTime'] = expecteddeparturetime.strftime("%H:%M:%S") 113 | row['TargetArrivalTime'] = row['TargetDepartureTime'] 114 | row['ExpectedArrivalTime'] = row['ExpectedDepartureTime'] 115 | row['TimingPointCode'] = avt['Station'] 116 | row['IsTimingStop'] = '1' 117 | row['UserStopCode'] = row['TimingPointCode'] 118 | row['StopAreaCode'] = row['TimingPointCode'] 119 | row['LinePublicNumber'] = str(linenumber(row['JourneyNumber'])) 120 | row['LinePlanningNumber'] = str(row['LinePublicNumber']) 121 | row['LineName'] =str(row ['LinePlanningNumber']) 122 | row['OperationDate'] = expecteddeparturetime.strftime("%Y-%m-%d") 123 | row['FortifyOrderNumber'] = '0' 124 | row['LocalServiceLevelCode'] = str(row['JourneyNumber']) 125 | row['NumberOfCoaches'] = '1' 126 | if (avt['TreinSoort'] == 'Stopbus i.p.v. Trein'): 127 | row['TransportType'] = 'BUS' 128 | else: 129 | row['TransportType'] = 'TRAIN' 130 | try: 131 | if row['JourneyNumber'] != None: 132 | if (int(row['JourneyNumber']) % 2 == 0): 133 | row['LineDirection'] = '2' 134 | else: 135 | row['LineDirection'] = '1' 136 | except: 137 | row['LineDirection'] = '0' 138 | row['UserStopOrderNumber'] = str(generateuserstopordernumber(targetdeparturetime)) 139 | return row 140 | 141 | 142 | 143 | while True: 144 | socks = dict(poller.poll()) 145 | 146 | if socks.get(subscribe_kv8) == zmq.POLLIN: 147 | print 'test' 148 | multipart = subscribe_kv8.recv_multipart() 149 | content = GzipFile('','r',0,StringIO(''.join(multipart[1:]))).read() 150 | c = ctx(content) 151 | print c.ctx 152 | if 'AVT' in c.ctx: 153 | rows = {} 154 | for row in c.ctx['AVT'].rows(): 155 | row['Station'] = c.ctx['Subscription'] 156 | kv8 = convertkv(row) 157 | pass_id = '_'.join([kv8['DataOwnerCode'], str(kv8['LocalServiceLevelCode']), str(kv8['LinePlanningNumber']), str(kv8['JourneyNumber']), str(kv8['FortifyOrderNumber']), kv8['UserStopCode'], str(kv8['UserStopOrderNumber'])]) 158 | rows[pass_id] = kv8 159 | print pass_id 160 | push.send_json(rows) 161 | 162 | -------------------------------------------------------------------------------- /rrd.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # rrd.py 4 | # Simple RRDTool wrapper 5 | # Copyright (c) 2008 Corey Goldberg (corey@goldb.org) 6 | # 7 | # Download the Windows version of RRDTool from: 8 | # http://www.gknw.net/mirror/rrdtool/ 9 | # 10 | # You may need these fonts if RRDTool throws an error when you graph: 11 | # http://dejavu.sourceforge.net/wiki/index.php/Main_Page 12 | 13 | 14 | import os 15 | import time 16 | import os.path 17 | 18 | class RRD(object): 19 | def __init__(self, rrd_name, vertical_label='test'): 20 | self.rrd_name = rrd_name 21 | self.vertical_label = vertical_label 22 | 23 | def create_rrd78(self): 24 | if not os.path.isfile(self.rrd_name): 25 | print 'write file' 26 | cmd_create = ''.join(( 27 | 'rrdtool create ', self.rrd_name, ' --step 60', 28 | ' DS:updates:DERIVE:15000:0:U', 29 | ' DS:passed:DERIVE:15000:0:U', 30 | ' RRA:AVERAGE:0.5:1:20000', 31 | )) 32 | cmd = os.popen4(cmd_create) 33 | cmd_output = cmd[1].read() 34 | for fd in cmd: fd.close() 35 | if len(cmd_output) > 0: 36 | raise RRDException, 'Unable to create RRD: ' + cmd_output 37 | 38 | def create_rrd(self, interval): 39 | interval = str(interval) 40 | interval_mins = float(interval) / 60 41 | heartbeat = str(int(interval) * 2) 42 | ds_string = ' DS:test:GAUGE:%s:U:U' % heartbeat 43 | cmd_create = ''.join(( 44 | 'rrdtool create ', self.rrd_name, ' --step ', interval, ds_string, 45 | ' RRA:AVERAGE:0.5:1:', str(int(4000 / interval_mins)), 46 | ' RRA:AVERAGE:0.5:', str(int(30 / interval_mins)), ':800', 47 | ' RRA:AVERAGE:0.5:', str(int(120 / interval_mins)), ':800', 48 | ' RRA:AVERAGE:0.5:', str(int(1440 / interval_mins)), ':800', 49 | )) 50 | cmd = os.popen4(cmd_create) 51 | cmd_output = cmd[1].read() 52 | for fd in cmd: fd.close() 53 | if len(cmd_output) > 0: 54 | raise RRDException, 'Unable to create RRD: ' + cmd_output 55 | 56 | def update78(self, msgcounter,passedcounter): 57 | cmd_update = 'rrdtool update '+self.rrd_name+' -t updates:passed N:'+msgcounter+':'+passedcounter 58 | cmd = os.popen4(cmd_update) 59 | cmd_output = cmd[1].read() 60 | for fd in cmd: fd.close() 61 | if len(cmd_output) > 0: 62 | raise RRDException, 'Unable to update RRD: ' + cmd_output 63 | 64 | def update(self, *values): 65 | values_args = ''.join([str(value) + ':' for value in values])[:-1] 66 | cmd_update = 'rrdtool update %s N:%s' % (self.rrd_name, values_args) 67 | cmd = os.popen4(cmd_update) 68 | cmd_output = cmd[1].read() 69 | for fd in cmd: fd.close() 70 | if len(cmd_output) > 0: 71 | raise RRDException, 'Unable to update RRD: ' + cmd_output 72 | 73 | def graph78(self): 74 | output_filename = self.rrd_name[:-4] + '.svg' 75 | width = '1200' 76 | height = '200' 77 | cur_date = time.strftime('%m/%d/%Y %H\:%M\:%S', time.localtime()) 78 | cmd_graph = 'rrdtool graph /var/ovapi/www/stats/' + output_filename \ 79 | + ' -a SVG --title '+self.rrd_name[:-4]+' --vertical-label "Updates per second"' \ 80 | + ' --width '+width+' --height '+height+' --start end-1w' \ 81 | + ' DEF:Updates='+self.rrd_name+':updates:AVERAGE' \ 82 | + ' DEF:PASSED='+self.rrd_name+':passed:AVERAGE' \ 83 | + ' AREA:Updates#FFFF00:Updates' \ 84 | + ' AREA:PASSED#009900:Timeupdates' 85 | cmd = os.popen4(cmd_graph) 86 | for fd in cmd: fd.close() 87 | 88 | def graph78stefan(self): 89 | output_filename = self.rrd_name[:-4] + '.png' 90 | width = '300' 91 | height = '75' 92 | cur_date = time.strftime('%m/%d/%Y %H\:%M\:%S', time.localtime()) 93 | cmd_graph = 'rrdtool graph /var/ovapi/www/stats/' + output_filename \ 94 | + ' -a PNG --vertical-label "'+self.rrd_name[:-4]+' upd/s"' \ 95 | + ' --width '+width+' --height '+height+' --start end-2d' \ 96 | + ' DEF:Updates='+self.rrd_name+':updates:AVERAGE' \ 97 | + ' DEF:PASSED='+self.rrd_name+':passed:AVERAGE' \ 98 | + ' AREA:Updates#FFFF00' \ 99 | + ' AREA:PASSED#009900' 100 | cmd = os.popen4(cmd_graph) 101 | for fd in cmd: fd.close() 102 | 103 | 104 | def graph(self, mins): 105 | start_time = 'now-%s' % (mins * 60) 106 | output_filename = self.rrd_name + '.png' 107 | end_time = 'now' 108 | ds_name = 'test' 109 | width = '300' 110 | height = '150' 111 | cur_date = time.strftime('%m/%d/%Y %H\:%M\:%S', time.localtime()) 112 | cmd_graph = 'rrdtool graph ' + output_filename + \ 113 | ' DEF:' + ds_name + '=' + self.rrd_name + ':' + ds_name + ':AVERAGE' + \ 114 | ' AREA:' + ds_name + '#FF0000' + \ 115 | ' VDEF:' + ds_name + 'last=' + ds_name + ',LAST' + \ 116 | ' VDEF:' + ds_name + 'avg=' + ds_name + ',AVERAGE' + \ 117 | ' COMMENT:"' + cur_date + '"' + \ 118 | ' GPRINT:' + ds_name + 'avg:" average=%6.2lf%S"' + \ 119 | ' --title="' + self.rrd_name +'"' + \ 120 | ' --vertical-label="' + self.vertical_label + '"' \ 121 | ' --start=' + start_time + \ 122 | ' --end=' + end_time + \ 123 | ' --width=' + width + \ 124 | ' --height=' + height + \ 125 | ' --lower-limit="0"' 126 | cmd = os.popen4(cmd_graph) 127 | for fd in cmd: fd.close() 128 | 129 | 130 | class RRDException(Exception): pass 131 | -------------------------------------------------------------------------------- /sphinx.conf: -------------------------------------------------------------------------------- 1 | # 2 | # Sphinx configuration file sample 3 | # 4 | # WARNING! While this sample file mentions all available options, 5 | # it contains (very) short helper descriptions only. Please refer to 6 | # doc/sphinx.html for details. 7 | # 8 | 9 | ############################################################################# 10 | ## data source definition 11 | ############################################################################# 12 | 13 | source haltes 14 | { 15 | # data source type. mandatory, no default value 16 | # known types are mysql, pgsql, mssql, xmlpipe, xmlpipe2, odbc 17 | type = pgsql 18 | 19 | ##################################################################### 20 | ## SQL settings (for 'mysql' and 'pgsql' types) 21 | ##################################################################### 22 | 23 | # some straightforward parameters for SQL source types 24 | sql_host = localhost 25 | sql_user = postgres 26 | sql_pass = postgres 27 | sql_db = haltes 28 | sql_port = 5433 # optional, default is 3306 29 | sql_field_string = TimingPointName 30 | sql_field_string = TimingPointTown 31 | sql_field_string = Name 32 | #sql_field_string = TimingPointCode 33 | # UNIX socket name 34 | # optional, default is empty (reuse client library defaults) 35 | # usually '/var/lib/mysql/mysql.sock' on Linux 36 | # usually '/tmp/mysql.sock' on FreeBSD 37 | # 38 | # sql_sock = /tmp/mysql.sock 39 | 40 | 41 | # MySQL specific client connection flags 42 | # optional, default is 0 43 | # 44 | # mysql_connect_flags = 32 # enable compression 45 | 46 | # MySQL specific SSL certificate settings 47 | # optional, defaults are empty 48 | # 49 | # mysql_ssl_cert = /etc/ssl/client-cert.pem 50 | # mysql_ssl_key = /etc/ssl/client-key.pem 51 | # mysql_ssl_ca = /etc/ssl/cacert.pem 52 | 53 | # MS SQL specific Windows authentication mode flag 54 | # MUST be in sync with charset_type index-level setting 55 | # optional, default is 0 56 | # 57 | # mssql_winauth = 1 # use currently logged on user credentials 58 | 59 | 60 | # MS SQL specific Unicode indexing flag 61 | # optional, default is 0 (request SBCS data) 62 | # 63 | # mssql_unicode = 1 # request Unicode data from server 64 | 65 | 66 | # ODBC specific DSN (data source name) 67 | # mandatory for odbc source type, no default value 68 | # 69 | # odbc_dsn = DBQ=C:\data;DefaultDir=C:\data;Driver={Microsoft Text Driver (*.txt; *.csv)}; 70 | # sql_query = SELECT id, data FROM documents.csv 71 | 72 | 73 | # pre-query, executed before the main fetch query 74 | # multi-value, optional, default is empty list of queries 75 | # 76 | # sql_query_pre = SET NAMES utf8 77 | # sql_query_pre = SET SESSION query_cache_type=OFF 78 | 79 | 80 | # main document fetch query 81 | # mandatory, integer document ID field MUST be the first selected column 82 | sql_query = select distinct on (name,timingpointtown) row_number() OVER (order by name),timingpointname,timingpointtown,name,latitude,longitude from timingpoint; 83 | 84 | # range query setup, query that must return min and max ID values 85 | # optional, default is empty 86 | # 87 | # sql_query will need to reference $start and $end boundaries 88 | # if using ranged query: 89 | # 90 | # sql_query = \ 91 | # SELECT doc.id, doc.id AS group, doc.title, doc.data \ 92 | # FROM documents doc \ 93 | # WHERE id>=$start AND id<=$end 94 | # 95 | # sql_query_range = SELECT MIN(id),MAX(id) FROM documents 96 | 97 | 98 | # range query step 99 | # optional, default is 1024 100 | # 101 | # sql_range_step = 1000 102 | 103 | 104 | # unsigned integer attribute declaration 105 | # multi-value (an arbitrary number of attributes is allowed), optional 106 | # optional bit size can be specified, default is 32 107 | # 108 | # sql_attr_uint = author_id 109 | # sql_attr_uint = forum_id:9 # 9 bits for forum_id 110 | # sql_attr_uint = timingpointcode 111 | #sql_field_string = timingpointcode 112 | #sql_field_string = timingpointname 113 | #sql_field_string = name 114 | #sql_field_string = timingpointtown 115 | # boolean attribute declaration 116 | # multi-value (an arbitrary number of attributes is allowed), optional 117 | # equivalent to sql_attr_uint with 1-bit size 118 | # 119 | # sql_attr_bool = is_deleted 120 | 121 | 122 | # bigint attribute declaration 123 | # multi-value (an arbitrary number of attributes is allowed), optional 124 | # declares a signed (unlike uint!) 64-bit attribute 125 | # 126 | # sql_attr_bigint = my_bigint_id 127 | 128 | # UNIX timestamp attribute declaration 129 | # multi-value (an arbitrary number of attributes is allowed), optional 130 | # similar to integer, but can also be used in date functions 131 | # 132 | # sql_attr_timestamp = posted_ts 133 | # sql_attr_timestamp = last_edited_ts 134 | sql_attr_timestamp = current_timestamp 135 | 136 | # string ordinal attribute declaration 137 | # multi-value (an arbitrary number of attributes is allowed), optional 138 | # sorts strings (bytewise), and stores their indexes in the sorted list 139 | # sorting by this attr is equivalent to sorting by the original strings 140 | # 141 | # sql_attr_str2ordinal = author_name 142 | 143 | 144 | # floating point attribute declaration 145 | # multi-value (an arbitrary number of attributes is allowed), optional 146 | # values are stored in single precision, 32-bit IEEE 754 format 147 | # 148 | sql_attr_float = latitude 149 | sql_attr_float = longitude 150 | 151 | 152 | # multi-valued attribute (MVA) attribute declaration 153 | # multi-value (an arbitrary number of attributes is allowed), optional 154 | # MVA values are variable length lists of unsigned 32-bit integers 155 | # 156 | # syntax is ATTR-TYPE ATTR-NAME 'from' SOURCE-TYPE [;QUERY] [;RANGE-QUERY] 157 | # ATTR-TYPE is 'uint' or 'timestamp' 158 | # SOURCE-TYPE is 'field', 'query', or 'ranged-query' 159 | # QUERY is SQL query used to fetch all ( docid, attrvalue ) pairs 160 | # RANGE-QUERY is SQL query used to fetch min and max ID values, similar to 'sql_query_range' 161 | # 162 | # sql_attr_multi = uint tag from query; SELECT id, tag FROM tags 163 | # sql_attr_multi = uint tag from ranged-query; \ 164 | # SELECT id, tag FROM tags WHERE id>=$start AND id<=$end; \ 165 | # SELECT MIN(id), MAX(id) FROM tags 166 | 167 | # post-query, executed on sql_query completion 168 | # optional, default is empty 169 | # 170 | # sql_query_post = 171 | 172 | 173 | # post-index-query, executed on successful indexing completion 174 | # optional, default is empty 175 | # $maxid expands to max document ID actually fetched from DB 176 | # 177 | # sql_query_post_index = REPLACE INTO counters ( id, val ) \ 178 | # VALUES ( 'max_indexed_id', $maxid ) 179 | 180 | 181 | # ranged query throttling, in milliseconds 182 | # optional, default is 0 which means no delay 183 | # enforces given delay before each query step 184 | sql_ranged_throttle = 0 185 | 186 | # document info query, ONLY for CLI search (ie. testing and debugging) 187 | # optional, default is empty 188 | # must contain $id macro and must fetch the document by that id 189 | # sql_query_info = SELECT * FROM documents WHERE id=$id 190 | sql_query_info = SELECT timingpointname, timingpointtown,name from timingpoint where timingpointcode = $timingpointcode; 191 | # kill-list query, fetches the document IDs for kill-list 192 | # k-list will suppress matches from preceding indexes in the same query 193 | # optional, default is empty 194 | # 195 | # sql_query_killlist = SELECT id FROM documents WHERE edited>=@last_reindex 196 | 197 | 198 | # columns to unpack on indexer side when indexing 199 | # multi-value, optional, default is empty list 200 | # 201 | # unpack_zlib = zlib_column 202 | # unpack_mysqlcompress = compressed_column 203 | # unpack_mysqlcompress = compressed_column_2 204 | 205 | 206 | # maximum unpacked length allowed in MySQL COMPRESS() unpacker 207 | # optional, default is 16M 208 | # 209 | # unpack_mysqlcompress_maxsize = 16M 210 | 211 | ##################################################################### 212 | ## xmlpipe settings 213 | ##################################################################### 214 | 215 | # type = xmlpipe 216 | 217 | # shell command to invoke xmlpipe stream producer 218 | # mandatory 219 | # 220 | # xmlpipe_command = cat /var/lib/sphinxsearch/test.xml 221 | 222 | ##################################################################### 223 | ## xmlpipe2 settings 224 | ##################################################################### 225 | 226 | # type = xmlpipe2 227 | # xmlpipe_command = cat /var/lib/sphinxsearch/test2.xml 228 | 229 | 230 | # xmlpipe2 field declaration 231 | # multi-value, optional, default is empty 232 | # 233 | # xmlpipe_field = subject 234 | # xmlpipe_field = content 235 | 236 | 237 | # xmlpipe2 attribute declaration 238 | # multi-value, optional, default is empty 239 | # all xmlpipe_attr_XXX options are fully similar to sql_attr_XXX 240 | # 241 | # xmlpipe_attr_timestamp = published 242 | # xmlpipe_attr_uint = author_id 243 | 244 | 245 | # perform UTF-8 validation, and filter out incorrect codes 246 | # avoids XML parser choking on non-UTF-8 documents 247 | # optional, default is 0 248 | # 249 | # xmlpipe_fixup_utf8 = 1 250 | } 251 | 252 | 253 | # inherited source example 254 | # 255 | # all the parameters are copied from the parent source, 256 | # and may then be overridden in this source definition 257 | #source haltesthrottled : haltes 258 | #{ 259 | # sql_field_string = timingpointname 260 | # sql_field_string = timingpointtown 261 | # sql_field_string = name 262 | # 263 | # sql_ranged_throttle = 100 264 | #} 265 | 266 | ############################################################################# 267 | ## index definition 268 | ############################################################################# 269 | 270 | # local index example 271 | # 272 | # this is an index which is stored locally in the filesystem 273 | # 274 | # all indexing-time options (such as morphology and charsets) 275 | # are configured per local index 276 | index haltes 277 | { 278 | # document source(s) to index 279 | # multi-value, mandatory 280 | # document IDs must be globally unique across all sources 281 | source = haltes 282 | 283 | # index files path and file name, without extension 284 | # mandatory, path must be writable, extensions will be auto-appended 285 | path = /var/ovapi/haltes/haltes 286 | 287 | # document attribute values (docinfo) storage mode 288 | # optional, default is 'extern' 289 | # known values are 'none', 'extern' and 'inline' 290 | docinfo = extern 291 | 292 | # memory locking for cached data (.spa and .spi), to prevent swapping 293 | # optional, default is 0 (do not mlock) 294 | # requires searchd to be run from root 295 | mlock = 0 296 | 297 | # a list of morphology preprocessors to apply 298 | # optional, default is empty 299 | # 300 | # builtin preprocessors are 'none', 'stem_en', 'stem_ru', 'stem_enru', 301 | # 'soundex', and 'metaphone'; additional preprocessors available from 302 | # libstemmer are 'libstemmer_XXX', where XXX is algorithm code 303 | # (see libstemmer_c/libstemmer/modules.txt) 304 | # 305 | # morphology = stem_en, stem_ru, soundex 306 | # morphology = libstemmer_german 307 | # morphology = libstemmer_sv 308 | morphology = none 309 | 310 | # minimum word length at which to enable stemming 311 | # optional, default is 1 (stem everything) 312 | # 313 | # min_stemming_len = 1 314 | 315 | 316 | # stopword files list (space separated) 317 | # optional, default is empty 318 | # contents are plain text, charset_table and stemming are both applied 319 | # 320 | # stopwords = /var/lib/sphinxsearch/data/stopwords.txt 321 | 322 | 323 | # wordforms file, in "mapfrom > mapto" plain text format 324 | # optional, default is empty 325 | # 326 | wordforms = /var/ovapi/haltes/wordforms.txt 327 | 328 | 329 | 330 | # tokenizing exceptions file 331 | # optional, default is empty 332 | # 333 | # plain text, case sensitive, space insensitive in map-from part 334 | # one "Map Several Words => ToASingleOne" entry per line 335 | # 336 | # exceptions = /var/lib/sphinxsearch/data/exceptions.txt 337 | 338 | 339 | # minimum indexed word length 340 | # default is 1 (index everything) 341 | min_word_len = 1 342 | 343 | # charset encoding type 344 | # optional, default is 'sbcs' 345 | # known types are 'sbcs' (Single Byte CharSet) and 'utf-8' 346 | charset_type = utf-8 347 | 348 | # charset definition and case folding rules "table" 349 | # optional, default value depends on charset_type 350 | # 351 | # defaults are configured to include English and Russian characters only 352 | # you need to change the table to include additional ones 353 | # this behavior MAY change in future versions 354 | # 355 | # 'sbcs' default value is 356 | # charset_table = 0..9, A..Z->a..z, _, a..z, U+A8->U+B8, U+B8, U+C0..U+DF->U+E0..U+FF, U+E0..U+FF 357 | # 358 | # 'utf-8' default value is 359 | # charset_table = 0..9, A..Z->a..z, _, a..z, U+410..U+42F->U+430..U+44F, U+430..U+44F 360 | 361 | 362 | # ignored characters list 363 | # optional, default value is empty 364 | # 365 | ignore_chars = U+00AD, - 366 | 367 | 368 | # minimum word prefix length to index 369 | # optional, default is 0 (do not index prefixes) 370 | # 371 | min_prefix_len = 0 372 | 373 | 374 | # minimum word infix length to index 375 | # optional, default is 0 (do not index infixes) 376 | # 377 | min_infix_len = 2 378 | 379 | 380 | # list of fields to limit prefix/infix indexing to 381 | # optional, default value is empty (index all fields in prefix/infix mode) 382 | # 383 | # prefix_fields = filename 384 | infix_fields = timingpointtown, name 385 | 386 | 387 | # enable star-syntax (wildcards) when searching prefix/infix indexes 388 | # known values are 0 and 1 389 | # optional, default is 0 (do not use wildcard syntax) 390 | # 391 | enable_star = 0 392 | 393 | 394 | # n-gram length to index, for CJK indexing 395 | # only supports 0 and 1 for now, other lengths to be implemented 396 | # optional, default is 0 (disable n-grams) 397 | # 398 | # ngram_len = 1 399 | 400 | 401 | # n-gram characters list, for CJK indexing 402 | # optional, default is empty 403 | # 404 | # ngram_chars = U+3000..U+2FA1F 405 | 406 | 407 | # phrase boundary characters list 408 | # optional, default is empty 409 | # 410 | # phrase_boundary = ., ?, !, U+2026 # horizontal ellipsis 411 | 412 | 413 | # phrase boundary word position increment 414 | # optional, default is 0 415 | # 416 | # phrase_boundary_step = 100 417 | 418 | 419 | # whether to strip HTML tags from incoming documents 420 | # known values are 0 (do not strip) and 1 (do strip) 421 | # optional, default is 0 422 | html_strip = 0 423 | 424 | # what HTML attributes to index if stripping HTML 425 | # optional, default is empty (do not index anything) 426 | # 427 | # html_index_attrs = img=alt,title; a=title; 428 | 429 | 430 | # what HTML elements contents to strip 431 | # optional, default is empty (do not strip element contents) 432 | # 433 | # html_remove_elements = style, script 434 | 435 | 436 | # whether to preopen index data files on startup 437 | # optional, default is 0 (do not preopen), searchd-only 438 | # 439 | # preopen = 1 440 | 441 | 442 | # whether to keep dictionary (.spi) on disk, or cache it in RAM 443 | # optional, default is 0 (cache in RAM), searchd-only 444 | # 445 | # ondisk_dict = 1 446 | 447 | 448 | # whether to enable in-place inversion (2x less disk, 90-95% speed) 449 | # optional, default is 0 (use separate temporary files), indexer-only 450 | # 451 | # inplace_enable = 1 452 | 453 | 454 | # in-place fine-tuning options 455 | # optional, defaults are listed below 456 | # 457 | # inplace_hit_gap = 0 # preallocated hitlist gap size 458 | # inplace_docinfo_gap = 0 # preallocated docinfo gap size 459 | # inplace_reloc_factor = 0.1 # relocation buffer size within arena 460 | # inplace_write_factor = 0.1 # write buffer size within arena 461 | 462 | 463 | # whether to index original keywords along with stemmed versions 464 | # enables "=exactform" operator to work 465 | # optional, default is 0 466 | # 467 | # index_exact_words = 1 468 | 469 | 470 | # position increment on overshort (less that min_word_len) words 471 | # optional, allowed values are 0 and 1, default is 1 472 | # 473 | # overshort_step = 1 474 | 475 | 476 | # position increment on stopword 477 | # optional, allowed values are 0 and 1, default is 1 478 | # 479 | # stopword_step = 1 480 | charset_table = 0..9, A..Z->a..z, a..z, \ 481 | U+00C0->a, U+00C1->a, U+00C2->a, U+00C3->a, U+00C4->a, U+00C5->a, U+00E0->a, U+00E1->a, U+00E2->a, U+00E3->a, U+00E4->a, U+00E5->a, U+0100->a, U+0101->a, U+0102->a, U+0103->a, U+010300->a, U+0104->a, U+0105->a, U+01CD->a, U+01CE->a, U+01DE->a, U+01DF->a, \ 482 | U+01E0->a, U+01E1->a, U+01FA->a, U+01FB->a, U+0200->a, U+0201->a, U+0202->a, U+0203->a, U+0226->a, U+0227->a, U+023A->a, U+0250->a, U+04D0->a, U+04D1->a, U+1D2C->a, U+1D43->a, U+1D44->a, U+1D8F->a, U+1E00->a, U+1E01->a, U+1E9A->a, U+1EA0->a, U+1EA1->a, \ 483 | U+1EA2->a, U+1EA3->a, U+1EA4->a, U+1EA5->a, U+1EA6->a, U+1EA7->a, U+1EA8->a, U+1EA9->a, U+1EAA->a, U+1EAB->a, U+1EAC->a, U+1EAD->a, U+1EAE->a, U+1EAF->a, U+1EB0->a, U+1EB1->a, U+1EB2->a, U+1EB3->a, U+1EB4->a, U+1EB5->a, U+1EB6->a, U+1EB7->a, U+2090->a, \ 484 | U+2C65->a, U+0180->b, U+0181->b, U+0182->b, U+0183->b, U+0243->b, U+0253->b, U+0299->b, U+16D2->b, U+1D03->b, U+1D2E->b, U+1D2F->b, U+1D47->b, U+1D6C->b, U+1D80->b, U+1E02->b, U+1E03->b, U+1E04->b, U+1E05->b, U+1E06->b, U+1E07->b, U+00C7->c, U+00E7->c, \ 485 | U+0106->c, U+0107->c, U+0108->c, U+0109->c, U+010A->c, U+010B->c, U+010C->c, U+010D->c, U+0187->c, U+0188->c, U+023B->c, U+023C->c, U+0255->c, U+0297->c, U+1D9C->c, U+1D9D->c, U+1E08->c, U+1E09->c, U+212D->c, U+2184->c, U+010E->d, U+010F->d, U+0110->d, \ 486 | U+0111->d, U+0189->d, U+018A->d, U+018B->d, U+018C->d, U+01C5->d, U+01F2->d, U+0221->d, U+0256->d, U+0257->d, U+1D05->d, U+1D30->d, U+1D48->d, U+1D6D->d, U+1D81->d, U+1D91->d, U+1E0A->d, U+1E0B->d, U+1E0C->d, U+1E0D->d, U+1E0E->d, U+1E0F->d, U+1E10->d, \ 487 | U+1E11->d, U+1E12->d, U+1E13->d, U+00C8->e, U+00C9->e, U+00CA->e, U+00CB->e, U+00E8->e, U+00E9->e, U+00EA->e, U+00EB->e, U+0112->e, U+0113->e, U+0114->e, U+0115->e, U+0116->e, U+0117->e, U+0118->e, U+0119->e, U+011A->e, U+011B->e, U+018E->e, U+0190->e, \ 488 | U+01DD->e, U+0204->e, U+0205->e, U+0206->e, U+0207->e, U+0228->e, U+0229->e, U+0246->e, U+0247->e, U+0258->e, U+025B->e, U+025C->e, U+025D->e, U+025E->e, U+029A->e, U+1D07->e, U+1D08->e, U+1D31->e, U+1D32->e, U+1D49->e, U+1D4B->e, U+1D4C->e, U+1D92->e, \ 489 | U+1D93->e, U+1D94->e, U+1D9F->e, U+1E14->e, U+1E15->e, U+1E16->e, U+1E17->e, U+1E18->e, U+1E19->e, U+1E1A->e, U+1E1B->e, U+1E1C->e, U+1E1D->e, U+1EB8->e, U+1EB9->e, U+1EBA->e, U+1EBB->e, U+1EBC->e, U+1EBD->e, U+1EBE->e, U+1EBF->e, U+1EC0->e, U+1EC1->e, \ 490 | U+1EC2->e, U+1EC3->e, U+1EC4->e, U+1EC5->e, U+1EC6->e, U+1EC7->e, U+2091->e, U+0191->f, U+0192->f, U+1D6E->f, U+1D82->f, U+1DA0->f, U+1E1E->f, U+1E1F->f, U+011C->g, U+011D->g, U+011E->g, U+011F->g, U+0120->g, U+0121->g, U+0122->g, U+0123->g, U+0193->g, \ 491 | U+01E4->g, U+01E5->g, U+01E6->g, U+01E7->g, U+01F4->g, U+01F5->g, U+0260->g, U+0261->g, U+0262->g, U+029B->g, U+1D33->g, U+1D4D->g, U+1D77->g, U+1D79->g, U+1D83->g, U+1DA2->g, U+1E20->g, U+1E21->g, U+0124->h, U+0125->h, U+0126->h, U+0127->h, U+021E->h, \ 492 | U+021F->h, U+0265->h, U+0266->h, U+029C->h, U+02AE->h, U+02AF->h, U+02B0->h, U+02B1->h, U+1D34->h, U+1DA3->h, U+1E22->h, U+1E23->h, U+1E24->h, U+1E25->h, U+1E26->h, U+1E27->h, U+1E28->h, U+1E29->h, U+1E2A->h, U+1E2B->h, U+1E96->h, U+210C->h, U+2C67->h, \ 493 | U+2C68->h, U+2C75->h, U+2C76->h, U+00CC->i, U+00CD->i, U+00CE->i, U+00CF->i, U+00EC->i, U+00ED->i, U+00EE->i, U+00EF->i, U+010309->i, U+0128->i, U+0129->i, U+012A->i, U+012B->i, U+012C->i, U+012D->i, U+012E->i, U+012F->i, U+0130->i, U+0131->i, U+0197->i, \ 494 | U+01CF->i, U+01D0->i, U+0208->i, U+0209->i, U+020A->i, U+020B->i, U+0268->i, U+026A->i, U+040D->i, U+0418->i, U+0419->i, U+0438->i, U+0439->i, U+0456->i, U+1D09->i, U+1D35->i, U+1D4E->i, U+1D62->i, U+1D7B->i, U+1D96->i, U+1DA4->i, U+1DA6->i, U+1DA7->i, \ 495 | U+1E2C->i, U+1E2D->i, U+1E2E->i, U+1E2F->i, U+1EC8->i, U+1EC9->i, U+1ECA->i, U+1ECB->i, U+2071->i, U+2111->i, U+0134->j, U+0135->j, U+01C8->j, U+01CB->j, U+01F0->j, U+0237->j, U+0248->j, U+0249->j, U+025F->j, U+0284->j, U+029D->j, U+02B2->j, U+1D0A->j, \ 496 | U+1D36->j, U+1DA1->j, U+1DA8->j, U+0136->k, U+0137->k, U+0198->k, U+0199->k, U+01E8->k, U+01E9->k, U+029E->k, U+1D0B->k, U+1D37->k, U+1D4F->k, U+1D84->k, U+1E30->k, U+1E31->k, U+1E32->k, U+1E33->k, U+1E34->k, U+1E35->k, U+2C69->k, U+2C6A->k, U+0139->l, \ 497 | U+013A->l, U+013B->l, U+013C->l, U+013D->l, U+013E->l, U+013F->l, U+0140->l, U+0141->l, U+0142->l, U+019A->l, U+01C8->l, U+0234->l, U+023D->l, U+026B->l, U+026C->l, U+026D->l, U+029F->l, U+02E1->l, U+1D0C->l, U+1D38->l, U+1D85->l, U+1DA9->l, U+1DAA->l, \ 498 | U+1DAB->l, U+1E36->l, U+1E37->l, U+1E38->l, U+1E39->l, U+1E3A->l, U+1E3B->l, U+1E3C->l, U+1E3D->l, U+2C60->l, U+2C61->l, U+2C62->l, U+019C->m, U+026F->m, U+0270->m, U+0271->m, U+1D0D->m, U+1D1F->m, U+1D39->m, U+1D50->m, U+1D5A->m, U+1D6F->m, U+1D86->m, \ 499 | U+1DAC->m, U+1DAD->m, U+1E3E->m, U+1E3F->m, U+1E40->m, U+1E41->m, U+1E42->m, U+1E43->m, U+00D1->n, U+00F1->n, U+0143->n, U+0144->n, U+0145->n, U+0146->n, U+0147->n, U+0148->n, U+0149->n, U+019D->n, U+019E->n, U+01CB->n, U+01F8->n, U+01F9->n, U+0220->n, \ 500 | U+0235->n, U+0272->n, U+0273->n, U+0274->n, U+1D0E->n, U+1D3A->n, U+1D3B->n, U+1D70->n, U+1D87->n, U+1DAE->n, U+1DAF->n, U+1DB0->n, U+1E44->n, U+1E45->n, U+1E46->n, U+1E47->n, U+1E48->n, U+1E49->n, U+1E4A->n, U+1E4B->n, U+207F->n, U+00D2->o, U+00D3->o, \ 501 | U+00D4->o, U+00D5->o, U+00D6->o, U+00D8->o, U+00F2->o, U+00F3->o, U+00F4->o, U+00F5->o, U+00F6->o, U+00F8->o, U+01030F->o, U+014C->o, U+014D->o, U+014E->o, U+014F->o, U+0150->o, U+0151->o, U+0186->o, U+019F->o, U+01A0->o, U+01A1->o, U+01D1->o, U+01D2->o, \ 502 | U+01EA->o, U+01EB->o, U+01EC->o, U+01ED->o, U+01FE->o, U+01FF->o, U+020C->o, U+020D->o, U+020E->o, U+020F->o, U+022A->o, U+022B->o, U+022C->o, U+022D->o, U+022E->o, U+022F->o, U+0230->o, U+0231->o, U+0254->o, U+0275->o, U+043E->o, U+04E6->o, U+04E7->o, \ 503 | U+04E8->o, U+04E9->o, U+04EA->o, U+04EB->o, U+1D0F->o, U+1D10->o, U+1D11->o, U+1D12->o, U+1D13->o, U+1D16->o, U+1D17->o, U+1D3C->o, U+1D52->o, U+1D53->o, U+1D54->o, U+1D55->o, U+1D97->o, U+1DB1->o, U+1E4C->o, U+1E4D->o, U+1E4E->o, U+1E4F->o, U+1E50->o, \ 504 | U+1E51->o, U+1E52->o, U+1E53->o, U+1ECC->o, U+1ECD->o, U+1ECE->o, U+1ECF->o, U+1ED0->o, U+1ED1->o, U+1ED2->o, U+1ED3->o, U+1ED4->o, U+1ED5->o, U+1ED6->o, U+1ED7->o, U+1ED8->o, U+1ED9->o, U+1EDA->o, U+1EDB->o, U+1EDC->o, U+1EDD->o, U+1EDE->o, U+1EDF->o, \ 505 | U+1EE0->o, U+1EE1->o, U+1EE2->o, U+1EE3->o, U+2092->o, U+2C9E->o, U+2C9F->o, U+01A4->p, U+01A5->p, U+1D18->p, U+1D3E->p, U+1D56->p, U+1D71->p, U+1D7D->p, U+1D88->p, U+1E54->p, U+1E55->p, U+1E56->p, U+1E57->p, U+2C63->p, U+024A->q, U+024B->q, U+02A0->q, \ 506 | U+0154->r, U+0155->r, U+0156->r, U+0157->r, U+0158->r, U+0159->r, U+0210->r, U+0211->r, U+0212->r, U+0213->r, U+024C->r, U+024D->r, U+0279->r, U+027A->r, U+027B->r, U+027C->r, U+027D->r, U+027E->r, U+027F->r, U+0280->r, U+0281->r, U+02B3->r, U+02B4->r, \ 507 | U+02B5->r, U+02B6->r, U+1D19->r, U+1D1A->r, U+1D3F->r, U+1D63->r, U+1D72->r, U+1D73->r, U+1D89->r, U+1DCA->r, U+1E58->r, U+1E59->r, U+1E5A->r, U+1E5B->r, U+1E5C->r, U+1E5D->r, U+1E5E->r, U+1E5F->r, U+211C->r, U+2C64->r, U+00DF->s, U+015A->s, U+015B->s, \ 508 | U+015C->s, U+015D->s, U+015E->s, U+015F->s, U+0160->s, U+0161->s, U+017F->s, U+0218->s, U+0219->s, U+023F->s, U+0282->s, U+02E2->s, U+1D74->s, U+1D8A->s, U+1DB3->s, U+1E60->s, U+1E61->s, U+1E62->s, U+1E63->s, U+1E64->s, U+1E65->s, U+1E66->s, U+1E67->s, \ 509 | U+1E68->s, U+1E69->s, U+1E9B->s, U+0162->t, U+0163->t, U+0164->t, U+0165->t, U+0166->t, U+0167->t, U+01AB->t, U+01AC->t, U+01AD->t, U+01AE->t, U+021A->t, U+021B->t, U+0236->t, U+023E->t, U+0287->t, U+0288->t, U+1D1B->t, U+1D40->t, U+1D57->t, U+1D75->t, \ 510 | U+1DB5->t, U+1E6A->t, U+1E6B->t, U+1E6C->t, U+1E6D->t, U+1E6E->t, U+1E6F->t, U+1E70->t, U+1E71->t, U+1E97->t, U+2C66->t, U+00D9->u, U+00DA->u, U+00DB->u, U+00DC->u, U+00F9->u, U+00FA->u, U+00FB->u, U+00FC->u, U+010316->u, U+0168->u, U+0169->u, U+016A->u, \ 511 | U+016B->u, U+016C->u, U+016D->u, U+016E->u, U+016F->u, U+0170->u, U+0171->u, U+0172->u, U+0173->u, U+01AF->u, U+01B0->u, U+01D3->u, U+01D4->u, U+01D5->u, U+01D6->u, U+01D7->u, U+01D8->u, U+01D9->u, U+01DA->u, U+01DB->u, U+01DC->u, U+0214->u, U+0215->u, \ 512 | U+0216->u, U+0217->u, U+0244->u, U+0289->u, U+1D1C->u, U+1D1D->u, U+1D1E->u, U+1D41->u, U+1D58->u, U+1D59->u, U+1D64->u, U+1D7E->u, U+1D99->u, U+1DB6->u, U+1DB8->u, U+1E72->u, U+1E73->u, U+1E74->u, U+1E75->u, U+1E76->u, U+1E77->u, U+1E78->u, U+1E79->u, \ 513 | U+1E7A->u, U+1E7B->u, U+1EE4->u, U+1EE5->u, U+1EE6->u, U+1EE7->u, U+1EE8->u, U+1EE9->u, U+1EEA->u, U+1EEB->u, U+1EEC->u, U+1EED->u, U+1EEE->u, U+1EEF->u, U+1EF0->u, U+1EF1->u, U+01B2->v, U+0245->v, U+028B->v, U+028C->v, U+1D20->v, U+1D5B->v, U+1D65->v, \ 514 | U+1D8C->v, U+1DB9->v, U+1DBA->v, U+1E7C->v, U+1E7D->v, U+1E7E->v, U+1E7F->v, U+2C74->v, U+0174->w, U+0175->w, U+028D->w, U+02B7->w, U+1D21->w, U+1D42->w, U+1E80->w, U+1E81->w, U+1E82->w, U+1E83->w, U+1E84->w, U+1E85->w, U+1E86->w, U+1E87->w, U+1E88->w, \ 515 | U+1E89->w, U+1E98->w, U+02E3->x, U+1D8D->x, U+1E8A->x, U+1E8B->x, U+1E8C->x, U+1E8D->x, U+2093->x, U+00DD->y, U+00FD->y, U+00FF->y, U+0176->y, U+0177->y, U+0178->y, U+01B3->y, U+01B4->y, U+0232->y, U+0233->y, U+024E->y, U+024F->y, U+028E->y, U+028F->y, \ 516 | U+02B8->y, U+1E8E->y, U+1E8F->y, U+1E99->y, U+1EF2->y, U+1EF3->y, U+1EF4->y, U+1EF5->y, U+1EF6->y, U+1EF7->y, U+1EF8->y, U+1EF9->y, U+0179->z, U+017A->z, U+017B->z, U+017C->z, U+017D->z, U+017E->z, U+01B5->z, U+01B6->z, U+0224->z, U+0225->z, U+0240->z, \ 517 | U+0290->z, U+0291->z, U+1D22->z, U+1D76->z, U+1D8E->z, U+1DBB->z, U+1DBC->z, U+1DBD->z, U+1E90->z, U+1E91->z, U+1E92->z, U+1E93->z, U+1E94->z, U+1E95->z, U+2128->z, U+2C6B->z, U+2C6C->z, U+00C6->U+00E6, U+01E2->U+00E6, U+01E3->U+00E6, U+01FC->U+00E6, \ 518 | U+01FD->U+00E6, U+1D01->U+00E6, U+1D02->U+00E6, U+1D2D->U+00E6, U+1D46->U+00E6, U+00E6, U+0622->U+0627, U+0623->U+0627, U+0624->U+0648, U+0625->U+0627, U+0626->U+064A, U+06C0->U+06D5, U+06C2->U+06C1, U+06D3->U+06D2, U+FB50->U+0671, U+FB51->U+0671, U+FB52->U+067B, \ 519 | U+FB53->U+067B, U+FB54->U+067B, U+FB56->U+067E, U+FB57->U+067E, U+FB58->U+067E, U+FB5A->U+0680, U+FB5B->U+0680, U+FB5C->U+0680, U+FB5E->U+067A, U+FB5F->U+067A, U+FB60->U+067A, U+FB62->U+067F, U+FB63->U+067F, U+FB64->U+067F, U+FB66->U+0679, U+FB67->U+0679, \ 520 | U+FB68->U+0679, U+FB6A->U+06A4, U+FB6B->U+06A4, U+FB6C->U+06A4, U+FB6E->U+06A6, U+FB6F->U+06A6, U+FB70->U+06A6, U+FB72->U+0684, U+FB73->U+0684, U+FB74->U+0684, U+FB76->U+0683, U+FB77->U+0683, U+FB78->U+0683, U+FB7A->U+0686, U+FB7B->U+0686, U+FB7C->U+0686, \ 521 | U+FB7E->U+0687, U+FB7F->U+0687, U+FB80->U+0687, U+FB82->U+068D, U+FB83->U+068D, U+FB84->U+068C, U+FB85->U+068C, U+FB86->U+068E, U+FB87->U+068E, U+FB88->U+0688, U+FB89->U+0688, U+FB8A->U+0698, U+FB8B->U+0698, U+FB8C->U+0691, U+FB8D->U+0691, U+FB8E->U+06A9, \ 522 | U+FB8F->U+06A9, U+FB90->U+06A9, U+FB92->U+06AF, U+FB93->U+06AF, U+FB94->U+06AF, U+FB96->U+06B3, U+FB97->U+06B3, U+FB98->U+06B3, U+FB9A->U+06B1, U+FB9B->U+06B1, U+FB9C->U+06B1, U+FB9E->U+06BA, U+FB9F->U+06BA, U+FBA0->U+06BB, U+FBA1->U+06BB, U+FBA2->U+06BB, \ 523 | U+FBA4->U+06C0, U+FBA5->U+06C0, U+FBA6->U+06C1, U+FBA7->U+06C1, U+FBA8->U+06C1, U+FBAA->U+06BE, U+FBAB->U+06BE, U+FBAC->U+06BE, U+FBAE->U+06D2, U+FBAF->U+06D2, U+FBB0->U+06D3, U+FBB1->U+06D3, U+FBD3->U+06AD, U+FBD4->U+06AD, U+FBD5->U+06AD, U+FBD7->U+06C7, \ 524 | U+FBD8->U+06C7, U+FBD9->U+06C6, U+FBDA->U+06C6, U+FBDB->U+06C8, U+FBDC->U+06C8, U+FBDD->U+0677, U+FBDE->U+06CB, U+FBDF->U+06CB, U+FBE0->U+06C5, U+FBE1->U+06C5, U+FBE2->U+06C9, U+FBE3->U+06C9, U+FBE4->U+06D0, U+FBE5->U+06D0, U+FBE6->U+06D0, U+FBE8->U+0649, \ 525 | U+FBFC->U+06CC, U+FBFD->U+06CC, U+FBFE->U+06CC, U+0621, U+0627..U+063A, U+0641..U+064A, U+0660..U+0669, U+066E, U+066F, U+0671..U+06BF, U+06C1, U+06C3..U+06D2, U+06D5, U+06EE..U+06FC, U+06FF, U+0750..U+076D, U+FB55, U+FB59, U+FB5D, U+FB61, U+FB65, U+FB69, \ 526 | U+FB6D, U+FB71, U+FB75, U+FB79, U+FB7D, U+FB81, U+FB91, U+FB95, U+FB99, U+FB9D, U+FBA3, U+FBA9, U+FBAD, U+FBD6, U+FBE7, U+FBE9, U+FBFF, U+0531..U+0556->U+0561..U+0586, U+0561..U+0586, U+0587, U+09DC->U+09A1, U+09DD->U+09A2, U+09DF->U+09AF, U+09F0->U+09AC, \ 527 | U+09F1->U+09AC, U+0985..U+0990, U+0993..U+09B0, U+09B2, U+09B6..U+09B9, U+09CE, U+09E0, U+09E1, U+09E6..U+09EF, U+F900->U+8C48, U+F901->U+66F4, U+F902->U+8ECA, U+F903->U+8CC8, U+F904->U+6ED1, U+F905->U+4E32, U+F906->U+53E5, U+F907->U+9F9C, U+F908->U+9F9C, \ 528 | U+F909->U+5951, U+F90A->U+91D1, U+F90B->U+5587, U+F90C->U+5948, U+F90D->U+61F6, U+F90E->U+7669, U+F90F->U+7F85, U+F910->U+863F, U+F911->U+87BA, U+F912->U+88F8, U+F913->U+908F, U+F914->U+6A02, U+F915->U+6D1B, U+F916->U+70D9, U+F917->U+73DE, U+F918->U+843D, \ 529 | U+F919->U+916A, U+F91A->U+99F1, U+F91B->U+4E82, U+F91C->U+5375, U+F91D->U+6B04, U+F91E->U+721B, U+F91F->U+862D, U+F920->U+9E1E, U+F921->U+5D50, U+F922->U+6FEB, U+F923->U+85CD, U+F924->U+8964, U+F925->U+62C9, U+F926->U+81D8, U+F927->U+881F, U+F928->U+5ECA, \ 530 | U+F929->U+6717, U+F92A->U+6D6A, U+F92B->U+72FC, U+F92C->U+90CE, U+F92D->U+4F86, U+F92E->U+51B7, U+F92F->U+52DE, U+F930->U+64C4, U+F931->U+6AD3, U+F932->U+7210, U+F933->U+76E7, U+F934->U+8001, U+F935->U+8606, U+F936->U+865C, U+F937->U+8DEF, U+F938->U+9732, \ 531 | U+F939->U+9B6F, U+F93A->U+9DFA, U+F93B->U+788C, U+F93C->U+797F, U+F93D->U+7DA0, U+F93E->U+83C9, U+F93F->U+9304, U+F940->U+9E7F, U+F941->U+8AD6, U+F942->U+58DF, U+F943->U+5F04, U+F944->U+7C60, U+F945->U+807E, U+F946->U+7262, U+F947->U+78CA, U+F948->U+8CC2, \ 532 | U+F949->U+96F7, U+F94A->U+58D8, U+F94B->U+5C62, U+F94C->U+6A13, U+F94D->U+6DDA, U+F94E->U+6F0F, U+F94F->U+7D2F, U+F950->U+7E37, U+F951->U+964B, U+F952->U+52D2, U+F953->U+808B, U+F954->U+51DC, U+F955->U+51CC, U+F956->U+7A1C, U+F957->U+7DBE, U+F958->U+83F1, \ 533 | U+F959->U+9675, U+F95A->U+8B80, U+F95B->U+62CF, U+F95C->U+6A02, U+F95D->U+8AFE, U+F95E->U+4E39, U+F95F->U+5BE7, U+F960->U+6012, U+F961->U+7387, U+F962->U+7570, U+F963->U+5317, U+F964->U+78FB, U+F965->U+4FBF, U+F966->U+5FA9, U+F967->U+4E0D, U+F968->U+6CCC, \ 534 | U+F969->U+6578, U+F96A->U+7D22, U+F96B->U+53C3, U+F96C->U+585E, U+F96D->U+7701, U+F96E->U+8449, U+F96F->U+8AAA, U+F970->U+6BBA, U+F971->U+8FB0, U+F972->U+6C88, U+F973->U+62FE, U+F974->U+82E5, U+F975->U+63A0, U+F976->U+7565, U+F977->U+4EAE, U+F978->U+5169, \ 535 | U+F979->U+51C9, U+F97A->U+6881, U+F97B->U+7CE7, U+F97C->U+826F, U+F97D->U+8AD2, U+F97E->U+91CF, U+F97F->U+52F5, U+F980->U+5442, U+F981->U+5973, U+F982->U+5EEC, U+F983->U+65C5, U+F984->U+6FFE, U+F985->U+792A, U+F986->U+95AD, U+F987->U+9A6A, U+F988->U+9E97, \ 536 | U+F989->U+9ECE, U+F98A->U+529B, U+F98B->U+66C6, U+F98C->U+6B77, U+F98D->U+8F62, U+F98E->U+5E74, U+F98F->U+6190, U+F990->U+6200, U+F991->U+649A, U+F992->U+6F23, U+F993->U+7149, U+F994->U+7489, U+F995->U+79CA, U+F996->U+7DF4, U+F997->U+806F, U+F998->U+8F26, \ 537 | U+F999->U+84EE, U+F99A->U+9023, U+F99B->U+934A, U+F99C->U+5217, U+F99D->U+52A3, U+F99E->U+54BD, U+F99F->U+70C8, U+F9A0->U+88C2, U+F9A1->U+8AAA, U+F9A2->U+5EC9, U+F9A3->U+5FF5, U+F9A4->U+637B, U+F9A5->U+6BAE, U+F9A6->U+7C3E, U+F9A7->U+7375, U+F9A8->U+4EE4, \ 538 | U+F9A9->U+56F9, U+F9AA->U+5BE7, U+F9AB->U+5DBA, U+F9AC->U+601C, U+F9AD->U+73B2, U+F9AE->U+7469, U+F9AF->U+7F9A, U+F9B0->U+8046, U+F9B1->U+9234, U+F9B2->U+96F6, U+F9B3->U+9748, U+F9B4->U+9818, U+F9B5->U+4F8B, U+F9B6->U+79AE, U+F9B7->U+91B4, U+F9B8->U+96B8, \ 539 | U+F9B9->U+60E1, U+F9BA->U+4E86, U+F9BB->U+50DA, U+F9BC->U+5BEE, U+F9BD->U+5C3F, U+F9BE->U+6599, U+F9BF->U+6A02, U+F9C0->U+71CE, U+F9C1->U+7642, U+F9C2->U+84FC, U+F9C3->U+907C, U+F9C4->U+9F8D, U+F9C5->U+6688, U+F9C6->U+962E, U+F9C7->U+5289, U+F9C8->U+677B, \ 540 | U+F9C9->U+67F3, U+F9CA->U+6D41, U+F9CB->U+6E9C, U+F9CC->U+7409, U+F9CD->U+7559, U+F9CE->U+786B, U+F9CF->U+7D10, U+F9D0->U+985E, U+F9D1->U+516D, U+F9D2->U+622E, U+F9D3->U+9678, U+F9D4->U+502B, U+F9D5->U+5D19, U+F9D6->U+6DEA, U+F9D7->U+8F2A, U+F9D8->U+5F8B, \ 541 | U+F9D9->U+6144, U+F9DA->U+6817, U+F9DB->U+7387, U+F9DC->U+9686, U+F9DD->U+5229, U+F9DE->U+540F, U+F9DF->U+5C65, U+F9E0->U+6613, U+F9E1->U+674E, U+F9E2->U+68A8, U+F9E3->U+6CE5, U+F9E4->U+7406, U+F9E5->U+75E2, U+F9E6->U+7F79, U+F9E7->U+88CF, U+F9E8->U+88E1, \ 542 | U+F9E9->U+91CC, U+F9EA->U+96E2, U+F9EB->U+533F, U+F9EC->U+6EBA, U+F9ED->U+541D, U+F9EE->U+71D0, U+F9EF->U+7498, U+F9F0->U+85FA, U+F9F1->U+96A3, U+F9F2->U+9C57, U+F9F3->U+9E9F, U+F9F4->U+6797, U+F9F5->U+6DCB, U+F9F6->U+81E8, U+F9F7->U+7ACB, U+F9F8->U+7B20, \ 543 | U+F9F9->U+7C92, U+F9FA->U+72C0, U+F9FB->U+7099, U+F9FC->U+8B58, U+F9FD->U+4EC0, U+F9FE->U+8336, U+F9FF->U+523A, U+FA00->U+5207, U+FA01->U+5EA6, U+FA02->U+62D3, U+FA03->U+7CD6, U+FA04->U+5B85, U+FA05->U+6D1E, U+FA06->U+66B4, U+FA07->U+8F3B, U+FA08->U+884C, \ 544 | U+FA09->U+964D, U+FA0A->U+898B, U+FA0B->U+5ED3, U+FA0C->U+5140, U+FA0D->U+55C0, U+FA10->U+585A, U+FA12->U+6674, U+FA15->U+51DE, U+FA16->U+732A, U+FA17->U+76CA, U+FA18->U+793C, U+FA19->U+795E, U+FA1A->U+7965, U+FA1B->U+798F, U+FA1C->U+9756, U+FA1D->U+7CBE, \ 545 | U+FA1E->U+7FBD, U+FA20->U+8612, U+FA22->U+8AF8, U+FA25->U+9038, U+FA26->U+90FD, U+FA2A->U+98EF, U+FA2B->U+98FC, U+FA2C->U+9928, U+FA2D->U+9DB4, U+FA30->U+4FAE, U+FA31->U+50E7, U+FA32->U+514D, U+FA33->U+52C9, U+FA34->U+52E4, U+FA35->U+5351, U+FA36->U+559D, \ 546 | U+FA37->U+5606, U+FA38->U+5668, U+FA39->U+5840, U+FA3A->U+58A8, U+FA3B->U+5C64, U+FA3C->U+5C6E, U+FA3D->U+6094, U+FA3E->U+6168, U+FA3F->U+618E, U+FA40->U+61F2, U+FA41->U+654F, U+FA42->U+65E2, U+FA43->U+6691, U+FA44->U+6885, U+FA45->U+6D77, U+FA46->U+6E1A, \ 547 | U+FA47->U+6F22, U+FA48->U+716E, U+FA49->U+722B, U+FA4A->U+7422, U+FA4B->U+7891, U+FA4C->U+793E, U+FA4D->U+7949, U+FA4E->U+7948, U+FA4F->U+7950, U+FA50->U+7956, U+FA51->U+795D, U+FA52->U+798D, U+FA53->U+798E, U+FA54->U+7A40, U+FA55->U+7A81, U+FA56->U+7BC0, \ 548 | U+FA57->U+7DF4, U+FA58->U+7E09, U+FA59->U+7E41, U+FA5A->U+7F72, U+FA5B->U+8005, U+FA5C->U+81ED, U+FA5D->U+8279, U+FA5E->U+8279, U+FA5F->U+8457, U+FA60->U+8910, U+FA61->U+8996, U+FA62->U+8B01, U+FA63->U+8B39, U+FA64->U+8CD3, U+FA65->U+8D08, U+FA66->U+8FB6, \ 549 | U+FA67->U+9038, U+FA68->U+96E3, U+FA69->U+97FF, U+FA6A->U+983B, U+FA70->U+4E26, U+FA71->U+51B5, U+FA72->U+5168, U+FA73->U+4F80, U+FA74->U+5145, U+FA75->U+5180, U+FA76->U+52C7, U+FA77->U+52FA, U+FA78->U+559D, U+FA79->U+5555, U+FA7A->U+5599, U+FA7B->U+55E2, \ 550 | U+FA7C->U+585A, U+FA7D->U+58B3, U+FA7E->U+5944, U+FA7F->U+5954, U+FA80->U+5A62, U+FA81->U+5B28, U+FA82->U+5ED2, U+FA83->U+5ED9, U+FA84->U+5F69, U+FA85->U+5FAD, U+FA86->U+60D8, U+FA87->U+614E, U+FA88->U+6108, U+FA89->U+618E, U+FA8A->U+6160, U+FA8B->U+61F2, \ 551 | U+FA8C->U+6234, U+FA8D->U+63C4, U+FA8E->U+641C, U+FA8F->U+6452, U+FA90->U+6556, U+FA91->U+6674, U+FA92->U+6717, U+FA93->U+671B, U+FA94->U+6756, U+FA95->U+6B79, U+FA96->U+6BBA, U+FA97->U+6D41, U+FA98->U+6EDB, U+FA99->U+6ECB, U+FA9A->U+6F22, U+FA9B->U+701E, \ 552 | U+FA9C->U+716E, U+FA9D->U+77A7, U+FA9E->U+7235, U+FA9F->U+72AF, U+FAA0->U+732A, U+FAA1->U+7471, U+FAA2->U+7506, U+FAA3->U+753B, U+FAA4->U+761D, U+FAA5->U+761F, U+FAA6->U+76CA, U+FAA7->U+76DB, U+FAA8->U+76F4, U+FAA9->U+774A, U+FAAA->U+7740, U+FAAB->U+78CC, \ 553 | U+FAAC->U+7AB1, U+FAAD->U+7BC0, U+FAAE->U+7C7B, U+FAAF->U+7D5B, U+FAB0->U+7DF4, U+FAB1->U+7F3E, U+FAB2->U+8005, U+FAB3->U+8352, U+FAB4->U+83EF, U+FAB5->U+8779, U+FAB6->U+8941, U+FAB7->U+8986, U+FAB8->U+8996, U+FAB9->U+8ABF, U+FABA->U+8AF8, U+FABB->U+8ACB, \ 554 | U+FABC->U+8B01, U+FABD->U+8AFE, U+FABE->U+8AED, U+FABF->U+8B39, U+FAC0->U+8B8A, U+FAC1->U+8D08, U+FAC2->U+8F38, U+FAC3->U+9072, U+FAC4->U+9199, U+FAC5->U+9276, U+FAC6->U+967C, U+FAC7->U+96E3, U+FAC8->U+9756, U+FAC9->U+97DB, U+FACA->U+97FF, U+FACB->U+980B, \ 555 | U+FACC->U+983B, U+FACD->U+9B12, U+FACE->U+9F9C, U+FACF->U+2284A, U+FAD0->U+22844, U+FAD1->U+233D5, U+FAD2->U+3B9D, U+FAD3->U+4018, U+FAD4->U+4039, U+FAD5->U+25249, U+FAD6->U+25CD0, U+FAD7->U+27ED3, U+FAD8->U+9F43, U+FAD9->U+9F8E, U+2F800->U+4E3D, U+2F801->U+4E38, \ 556 | U+2F802->U+4E41, U+2F803->U+20122, U+2F804->U+4F60, U+2F805->U+4FAE, U+2F806->U+4FBB, U+2F807->U+5002, U+2F808->U+507A, U+2F809->U+5099, U+2F80A->U+50E7, U+2F80B->U+50CF, U+2F80C->U+349E, U+2F80D->U+2063A, U+2F80E->U+514D, U+2F80F->U+5154, U+2F810->U+5164, \ 557 | U+2F811->U+5177, U+2F812->U+2051C, U+2F813->U+34B9, U+2F814->U+5167, U+2F815->U+518D, U+2F816->U+2054B, U+2F817->U+5197, U+2F818->U+51A4, U+2F819->U+4ECC, U+2F81A->U+51AC, U+2F81B->U+51B5, U+2F81C->U+291DF, U+2F81D->U+51F5, U+2F81E->U+5203, U+2F81F->U+34DF, \ 558 | U+2F820->U+523B, U+2F821->U+5246, U+2F822->U+5272, U+2F823->U+5277, U+2F824->U+3515, U+2F825->U+52C7, U+2F826->U+52C9, U+2F827->U+52E4, U+2F828->U+52FA, U+2F829->U+5305, U+2F82A->U+5306, U+2F82B->U+5317, U+2F82C->U+5349, U+2F82D->U+5351, U+2F82E->U+535A, \ 559 | U+2F82F->U+5373, U+2F830->U+537D, U+2F831->U+537F, U+2F832->U+537F, U+2F833->U+537F, U+2F834->U+20A2C, U+2F835->U+7070, U+2F836->U+53CA, U+2F837->U+53DF, U+2F838->U+20B63, U+2F839->U+53EB, U+2F83A->U+53F1, U+2F83B->U+5406, U+2F83C->U+549E, U+2F83D->U+5438, \ 560 | U+2F83E->U+5448, U+2F83F->U+5468, U+2F840->U+54A2, U+2F841->U+54F6, U+2F842->U+5510, U+2F843->U+5553, U+2F844->U+5563, U+2F845->U+5584, U+2F846->U+5584, U+2F847->U+5599, U+2F848->U+55AB, U+2F849->U+55B3, U+2F84A->U+55C2, U+2F84B->U+5716, U+2F84C->U+5606, \ 561 | U+2F84D->U+5717, U+2F84E->U+5651, U+2F84F->U+5674, U+2F850->U+5207, U+2F851->U+58EE, U+2F852->U+57CE, U+2F853->U+57F4, U+2F854->U+580D, U+2F855->U+578B, U+2F856->U+5832, U+2F857->U+5831, U+2F858->U+58AC, U+2F859->U+214E4, U+2F85A->U+58F2, U+2F85B->U+58F7, \ 562 | U+2F85C->U+5906, U+2F85D->U+591A, U+2F85E->U+5922, U+2F85F->U+5962, U+2F860->U+216A8, U+2F861->U+216EA, U+2F862->U+59EC, U+2F863->U+5A1B, U+2F864->U+5A27, U+2F865->U+59D8, U+2F866->U+5A66, U+2F867->U+36EE, U+2F868->U+36FC, U+2F869->U+5B08, U+2F86A->U+5B3E, \ 563 | U+2F86B->U+5B3E, U+2F86C->U+219C8, U+2F86D->U+5BC3, U+2F86E->U+5BD8, U+2F86F->U+5BE7, U+2F870->U+5BF3, U+2F871->U+21B18, U+2F872->U+5BFF, U+2F873->U+5C06, U+2F874->U+5F53, U+2F875->U+5C22, U+2F876->U+3781, U+2F877->U+5C60, U+2F878->U+5C6E, U+2F879->U+5CC0, \ 564 | U+2F87A->U+5C8D, U+2F87B->U+21DE4, U+2F87C->U+5D43, U+2F87D->U+21DE6, U+2F87E->U+5D6E, U+2F87F->U+5D6B, U+2F880->U+5D7C, U+2F881->U+5DE1, U+2F882->U+5DE2, U+2F883->U+382F, U+2F884->U+5DFD, U+2F885->U+5E28, U+2F886->U+5E3D, U+2F887->U+5E69, U+2F888->U+3862, \ 565 | U+2F889->U+22183, U+2F88A->U+387C, U+2F88B->U+5EB0, U+2F88C->U+5EB3, U+2F88D->U+5EB6, U+2F88E->U+5ECA, U+2F88F->U+2A392, U+2F890->U+5EFE, U+2F891->U+22331, U+2F892->U+22331, U+2F893->U+8201, U+2F894->U+5F22, U+2F895->U+5F22, U+2F896->U+38C7, U+2F897->U+232B8, \ 566 | U+2F898->U+261DA, U+2F899->U+5F62, U+2F89A->U+5F6B, U+2F89B->U+38E3, U+2F89C->U+5F9A, U+2F89D->U+5FCD, U+2F89E->U+5FD7, U+2F89F->U+5FF9, U+2F8A0->U+6081, U+2F8A1->U+393A, U+2F8A2->U+391C, U+2F8A3->U+6094, U+2F8A4->U+226D4, U+2F8A5->U+60C7, U+2F8A6->U+6148, \ 567 | U+2F8A7->U+614C, U+2F8A8->U+614E, U+2F8A9->U+614C, U+2F8AA->U+617A, U+2F8AB->U+618E, U+2F8AC->U+61B2, U+2F8AD->U+61A4, U+2F8AE->U+61AF, U+2F8AF->U+61DE, U+2F8B0->U+61F2, U+2F8B1->U+61F6, U+2F8B2->U+6210, U+2F8B3->U+621B, U+2F8B4->U+625D, U+2F8B5->U+62B1, \ 568 | U+2F8B6->U+62D4, U+2F8B7->U+6350, U+2F8B8->U+22B0C, U+2F8B9->U+633D, U+2F8BA->U+62FC, U+2F8BB->U+6368, U+2F8BC->U+6383, U+2F8BD->U+63E4, U+2F8BE->U+22BF1, U+2F8BF->U+6422, U+2F8C0->U+63C5, U+2F8C1->U+63A9, U+2F8C2->U+3A2E, U+2F8C3->U+6469, U+2F8C4->U+647E, \ 569 | U+2F8C5->U+649D, U+2F8C6->U+6477, U+2F8C7->U+3A6C, U+2F8C8->U+654F, U+2F8C9->U+656C, U+2F8CA->U+2300A, U+2F8CB->U+65E3, U+2F8CC->U+66F8, U+2F8CD->U+6649, U+2F8CE->U+3B19, U+2F8CF->U+6691, U+2F8D0->U+3B08, U+2F8D1->U+3AE4, U+2F8D2->U+5192, U+2F8D3->U+5195, \ 570 | U+2F8D4->U+6700, U+2F8D5->U+669C, U+2F8D6->U+80AD, U+2F8D7->U+43D9, U+2F8D8->U+6717, U+2F8D9->U+671B, U+2F8DA->U+6721, U+2F8DB->U+675E, U+2F8DC->U+6753, U+2F8DD->U+233C3, U+2F8DE->U+3B49, U+2F8DF->U+67FA, U+2F8E0->U+6785, U+2F8E1->U+6852, U+2F8E2->U+6885, \ 571 | U+2F8E3->U+2346D, U+2F8E4->U+688E, U+2F8E5->U+681F, U+2F8E6->U+6914, U+2F8E7->U+3B9D, U+2F8E8->U+6942, U+2F8E9->U+69A3, U+2F8EA->U+69EA, U+2F8EB->U+6AA8, U+2F8EC->U+236A3, U+2F8ED->U+6ADB, U+2F8EE->U+3C18, U+2F8EF->U+6B21, U+2F8F0->U+238A7, U+2F8F1->U+6B54, \ 572 | U+2F8F2->U+3C4E, U+2F8F3->U+6B72, U+2F8F4->U+6B9F, U+2F8F5->U+6BBA, U+2F8F6->U+6BBB, U+2F8F7->U+23A8D, U+2F8F8->U+21D0B, U+2F8F9->U+23AFA, U+2F8FA->U+6C4E, U+2F8FB->U+23CBC, U+2F8FC->U+6CBF, U+2F8FD->U+6CCD, U+2F8FE->U+6C67, U+2F8FF->U+6D16, U+2F900->U+6D3E, \ 573 | U+2F901->U+6D77, U+2F902->U+6D41, U+2F903->U+6D69, U+2F904->U+6D78, U+2F905->U+6D85, U+2F906->U+23D1E, U+2F907->U+6D34, U+2F908->U+6E2F, U+2F909->U+6E6E, U+2F90A->U+3D33, U+2F90B->U+6ECB, U+2F90C->U+6EC7, U+2F90D->U+23ED1, U+2F90E->U+6DF9, U+2F90F->U+6F6E, \ 574 | U+2F910->U+23F5E, U+2F911->U+23F8E, U+2F912->U+6FC6, U+2F913->U+7039, U+2F914->U+701E, U+2F915->U+701B, U+2F916->U+3D96, U+2F917->U+704A, U+2F918->U+707D, U+2F919->U+7077, U+2F91A->U+70AD, U+2F91B->U+20525, U+2F91C->U+7145, U+2F91D->U+24263, U+2F91E->U+719C, \ 575 | U+2F91F->U+243AB, U+2F920->U+7228, U+2F921->U+7235, U+2F922->U+7250, U+2F923->U+24608, U+2F924->U+7280, U+2F925->U+7295, U+2F926->U+24735, U+2F927->U+24814, U+2F928->U+737A, U+2F929->U+738B, U+2F92A->U+3EAC, U+2F92B->U+73A5, U+2F92C->U+3EB8, U+2F92D->U+3EB8, \ 576 | U+2F92E->U+7447, U+2F92F->U+745C, U+2F930->U+7471, U+2F931->U+7485, U+2F932->U+74CA, U+2F933->U+3F1B, U+2F934->U+7524, U+2F935->U+24C36, U+2F936->U+753E, U+2F937->U+24C92, U+2F938->U+7570, U+2F939->U+2219F, U+2F93A->U+7610, U+2F93B->U+24FA1, U+2F93C->U+24FB8, \ 577 | U+2F93D->U+25044, U+2F93E->U+3FFC, U+2F93F->U+4008, U+2F940->U+76F4, U+2F941->U+250F3, U+2F942->U+250F2, U+2F943->U+25119, U+2F944->U+25133, U+2F945->U+771E, U+2F946->U+771F, U+2F947->U+771F, U+2F948->U+774A, U+2F949->U+4039, U+2F94A->U+778B, U+2F94B->U+4046, \ 578 | U+2F94C->U+4096, U+2F94D->U+2541D, U+2F94E->U+784E, U+2F94F->U+788C, U+2F950->U+78CC, U+2F951->U+40E3, U+2F952->U+25626, U+2F953->U+7956, U+2F954->U+2569A, U+2F955->U+256C5, U+2F956->U+798F, U+2F957->U+79EB, U+2F958->U+412F, U+2F959->U+7A40, U+2F95A->U+7A4A, \ 579 | U+2F95B->U+7A4F, U+2F95C->U+2597C, U+2F95D->U+25AA7, U+2F95E->U+25AA7, U+2F95F->U+7AEE, U+2F960->U+4202, U+2F961->U+25BAB, U+2F962->U+7BC6, U+2F963->U+7BC9, U+2F964->U+4227, U+2F965->U+25C80, U+2F966->U+7CD2, U+2F967->U+42A0, U+2F968->U+7CE8, U+2F969->U+7CE3, \ 580 | U+2F96A->U+7D00, U+2F96B->U+25F86, U+2F96C->U+7D63, U+2F96D->U+4301, U+2F96E->U+7DC7, U+2F96F->U+7E02, U+2F970->U+7E45, U+2F971->U+4334, U+2F972->U+26228, U+2F973->U+26247, U+2F974->U+4359, U+2F975->U+262D9, U+2F976->U+7F7A, U+2F977->U+2633E, U+2F978->U+7F95, \ 581 | U+2F979->U+7FFA, U+2F97A->U+8005, U+2F97B->U+264DA, U+2F97C->U+26523, U+2F97D->U+8060, U+2F97E->U+265A8, U+2F97F->U+8070, U+2F980->U+2335F, U+2F981->U+43D5, U+2F982->U+80B2, U+2F983->U+8103, U+2F984->U+440B, U+2F985->U+813E, U+2F986->U+5AB5, U+2F987->U+267A7, \ 582 | U+2F988->U+267B5, U+2F989->U+23393, U+2F98A->U+2339C, U+2F98B->U+8201, U+2F98C->U+8204, U+2F98D->U+8F9E, U+2F98E->U+446B, U+2F98F->U+8291, U+2F990->U+828B, U+2F991->U+829D, U+2F992->U+52B3, U+2F993->U+82B1, U+2F994->U+82B3, U+2F995->U+82BD, U+2F996->U+82E6, \ 583 | U+2F997->U+26B3C, U+2F998->U+82E5, U+2F999->U+831D, U+2F99A->U+8363, U+2F99B->U+83AD, U+2F99C->U+8323, U+2F99D->U+83BD, U+2F99E->U+83E7, U+2F99F->U+8457, U+2F9A0->U+8353, U+2F9A1->U+83CA, U+2F9A2->U+83CC, U+2F9A3->U+83DC, U+2F9A4->U+26C36, U+2F9A5->U+26D6B, \ 584 | U+2F9A6->U+26CD5, U+2F9A7->U+452B, U+2F9A8->U+84F1, U+2F9A9->U+84F3, U+2F9AA->U+8516, U+2F9AB->U+273CA, U+2F9AC->U+8564, U+2F9AD->U+26F2C, U+2F9AE->U+455D, U+2F9AF->U+4561, U+2F9B0->U+26FB1, U+2F9B1->U+270D2, U+2F9B2->U+456B, U+2F9B3->U+8650, U+2F9B4->U+865C, \ 585 | U+2F9B5->U+8667, U+2F9B6->U+8669, U+2F9B7->U+86A9, U+2F9B8->U+8688, U+2F9B9->U+870E, U+2F9BA->U+86E2, U+2F9BB->U+8779, U+2F9BC->U+8728, U+2F9BD->U+876B, U+2F9BE->U+8786, U+2F9BF->U+45D7, U+2F9C0->U+87E1, U+2F9C1->U+8801, U+2F9C2->U+45F9, U+2F9C3->U+8860, \ 586 | U+2F9C4->U+8863, U+2F9C5->U+27667, U+2F9C6->U+88D7, U+2F9C7->U+88DE, U+2F9C8->U+4635, U+2F9C9->U+88FA, U+2F9CA->U+34BB, U+2F9CB->U+278AE, U+2F9CC->U+27966, U+2F9CD->U+46BE, U+2F9CE->U+46C7, U+2F9CF->U+8AA0, U+2F9D0->U+8AED, U+2F9D1->U+8B8A, U+2F9D2->U+8C55, \ 587 | U+2F9D3->U+27CA8, U+2F9D4->U+8CAB, U+2F9D5->U+8CC1, U+2F9D6->U+8D1B, U+2F9D7->U+8D77, U+2F9D8->U+27F2F, U+2F9D9->U+20804, U+2F9DA->U+8DCB, U+2F9DB->U+8DBC, U+2F9DC->U+8DF0, U+2F9DD->U+208DE, U+2F9DE->U+8ED4, U+2F9DF->U+8F38, U+2F9E0->U+285D2, U+2F9E1->U+285ED, \ 588 | U+2F9E2->U+9094, U+2F9E3->U+90F1, U+2F9E4->U+9111, U+2F9E5->U+2872E, U+2F9E6->U+911B, U+2F9E7->U+9238, U+2F9E8->U+92D7, U+2F9E9->U+92D8, U+2F9EA->U+927C, U+2F9EB->U+93F9, U+2F9EC->U+9415, U+2F9ED->U+28BFA, U+2F9EE->U+958B, U+2F9EF->U+4995, U+2F9F0->U+95B7, \ 589 | U+2F9F1->U+28D77, U+2F9F2->U+49E6, U+2F9F3->U+96C3, U+2F9F4->U+5DB2, U+2F9F5->U+9723, U+2F9F6->U+29145, U+2F9F7->U+2921A, U+2F9F8->U+4A6E, U+2F9F9->U+4A76, U+2F9FA->U+97E0, U+2F9FB->U+2940A, U+2F9FC->U+4AB2, U+2F9FD->U+29496, U+2F9FE->U+980B, U+2F9FF->U+980B, \ 590 | U+2FA00->U+9829, U+2FA01->U+295B6, U+2FA02->U+98E2, U+2FA03->U+4B33, U+2FA04->U+9929, U+2FA05->U+99A7, U+2FA06->U+99C2, U+2FA07->U+99FE, U+2FA08->U+4BCE, U+2FA09->U+29B30, U+2FA0A->U+9B12, U+2FA0B->U+9C40, U+2FA0C->U+9CFD, U+2FA0D->U+4CCE, U+2FA0E->U+4CED, \ 591 | U+2FA0F->U+9D67, U+2FA10->U+2A0CE, U+2FA11->U+4CF8, U+2FA12->U+2A105, U+2FA13->U+2A20E, U+2FA14->U+2A291, U+2FA15->U+9EBB, U+2FA16->U+4D56, U+2FA17->U+9EF9, U+2FA18->U+9EFE, U+2FA19->U+9F05, U+2FA1A->U+9F0F, U+2FA1B->U+9F16, U+2FA1C->U+9F3B, U+2FA1D->U+2A600, \ 592 | U+2F00->U+4E00, U+2F01->U+4E28, U+2F02->U+4E36, U+2F03->U+4E3F, U+2F04->U+4E59, U+2F05->U+4E85, U+2F06->U+4E8C, U+2F07->U+4EA0, U+2F08->U+4EBA, U+2F09->U+513F, U+2F0A->U+5165, U+2F0B->U+516B, U+2F0C->U+5182, U+2F0D->U+5196, U+2F0E->U+51AB, U+2F0F->U+51E0, \ 593 | U+2F10->U+51F5, U+2F11->U+5200, U+2F12->U+529B, U+2F13->U+52F9, U+2F14->U+5315, U+2F15->U+531A, U+2F16->U+5338, U+2F17->U+5341, U+2F18->U+535C, U+2F19->U+5369, U+2F1A->U+5382, U+2F1B->U+53B6, U+2F1C->U+53C8, U+2F1D->U+53E3, U+2F1E->U+56D7, U+2F1F->U+571F, \ 594 | U+2F20->U+58EB, U+2F21->U+5902, U+2F22->U+590A, U+2F23->U+5915, U+2F24->U+5927, U+2F25->U+5973, U+2F26->U+5B50, U+2F27->U+5B80, U+2F28->U+5BF8, U+2F29->U+5C0F, U+2F2A->U+5C22, U+2F2B->U+5C38, U+2F2C->U+5C6E, U+2F2D->U+5C71, U+2F2E->U+5DDB, U+2F2F->U+5DE5, \ 595 | U+2F30->U+5DF1, U+2F31->U+5DFE, U+2F32->U+5E72, U+2F33->U+5E7A, U+2F34->U+5E7F, U+2F35->U+5EF4, U+2F36->U+5EFE, U+2F37->U+5F0B, U+2F38->U+5F13, U+2F39->U+5F50, U+2F3A->U+5F61, U+2F3B->U+5F73, U+2F3C->U+5FC3, U+2F3D->U+6208, U+2F3E->U+6236, U+2F3F->U+624B, \ 596 | U+2F40->U+652F, U+2F41->U+6534, U+2F42->U+6587, U+2F43->U+6597, U+2F44->U+65A4, U+2F45->U+65B9, U+2F46->U+65E0, U+2F47->U+65E5, U+2F48->U+66F0, U+2F49->U+6708, U+2F4A->U+6728, U+2F4B->U+6B20, U+2F4C->U+6B62, U+2F4D->U+6B79, U+2F4E->U+6BB3, U+2F4F->U+6BCB, \ 597 | U+2F50->U+6BD4, U+2F51->U+6BDB, U+2F52->U+6C0F, U+2F53->U+6C14, U+2F54->U+6C34, U+2F55->U+706B, U+2F56->U+722A, U+2F57->U+7236, U+2F58->U+723B, U+2F59->U+723F, U+2F5A->U+7247, U+2F5B->U+7259, U+2F5C->U+725B, U+2F5D->U+72AC, U+2F5E->U+7384, U+2F5F->U+7389, \ 598 | U+2F60->U+74DC, U+2F61->U+74E6, U+2F62->U+7518, U+2F63->U+751F, U+2F64->U+7528, U+2F65->U+7530, U+2F66->U+758B, U+2F67->U+7592, U+2F68->U+7676, U+2F69->U+767D, U+2F6A->U+76AE, U+2F6B->U+76BF, U+2F6C->U+76EE, U+2F6D->U+77DB, U+2F6E->U+77E2, U+2F6F->U+77F3, \ 599 | U+2F70->U+793A, U+2F71->U+79B8, U+2F72->U+79BE, U+2F73->U+7A74, U+2F74->U+7ACB, U+2F75->U+7AF9, U+2F76->U+7C73, U+2F77->U+7CF8, U+2F78->U+7F36, U+2F79->U+7F51, U+2F7A->U+7F8A, U+2F7B->U+7FBD, U+2F7C->U+8001, U+2F7D->U+800C, U+2F7E->U+8012, U+2F7F->U+8033, \ 600 | U+2F80->U+807F, U+2F81->U+8089, U+2F82->U+81E3, U+2F83->U+81EA, U+2F84->U+81F3, U+2F85->U+81FC, U+2F86->U+820C, U+2F87->U+821B, U+2F88->U+821F, U+2F89->U+826E, U+2F8A->U+8272, U+2F8B->U+8278, U+2F8C->U+864D, U+2F8D->U+866B, U+2F8E->U+8840, U+2F8F->U+884C, \ 601 | U+2F90->U+8863, U+2F91->U+897E, U+2F92->U+898B, U+2F93->U+89D2, U+2F94->U+8A00, U+2F95->U+8C37, U+2F96->U+8C46, U+2F97->U+8C55, U+2F98->U+8C78, U+2F99->U+8C9D, U+2F9A->U+8D64, U+2F9B->U+8D70, U+2F9C->U+8DB3, U+2F9D->U+8EAB, U+2F9E->U+8ECA, U+2F9F->U+8F9B, \ 602 | U+2FA0->U+8FB0, U+2FA1->U+8FB5, U+2FA2->U+9091, U+2FA3->U+9149, U+2FA4->U+91C6, U+2FA5->U+91CC, U+2FA6->U+91D1, U+2FA7->U+9577, U+2FA8->U+9580, U+2FA9->U+961C, U+2FAA->U+96B6, U+2FAB->U+96B9, U+2FAC->U+96E8, U+2FAD->U+9751, U+2FAE->U+975E, U+2FAF->U+9762, \ 603 | U+2FB0->U+9769, U+2FB1->U+97CB, U+2FB2->U+97ED, U+2FB3->U+97F3, U+2FB4->U+9801, U+2FB5->U+98A8, U+2FB6->U+98DB, U+2FB7->U+98DF, U+2FB8->U+9996, U+2FB9->U+9999, U+2FBA->U+99AC, U+2FBB->U+9AA8, U+2FBC->U+9AD8, U+2FBD->U+9ADF, U+2FBE->U+9B25, U+2FBF->U+9B2F, \ 604 | U+2FC0->U+9B32, U+2FC1->U+9B3C, U+2FC2->U+9B5A, U+2FC3->U+9CE5, U+2FC4->U+9E75, U+2FC5->U+9E7F, U+2FC6->U+9EA5, U+2FC7->U+9EBB, U+2FC8->U+9EC3, U+2FC9->U+9ECD, U+2FCA->U+9ED1, U+2FCB->U+9EF9, U+2FCC->U+9EFD, U+2FCD->U+9F0E, U+2FCE->U+9F13, U+2FCF->U+9F20, \ 605 | U+2FD0->U+9F3B, U+2FD1->U+9F4A, U+2FD2->U+9F52, U+2FD3->U+9F8D, U+2FD4->U+9F9C, U+2FD5->U+9FA0, U+3042->U+3041, U+3044->U+3043, U+3046->U+3045, U+3048->U+3047, U+304A->U+3049, U+304C->U+304B, U+304E->U+304D, U+3050->U+304F, U+3052->U+3051, U+3054->U+3053, \ 606 | U+3056->U+3055, U+3058->U+3057, U+305A->U+3059, U+305C->U+305B, U+305E->U+305D, U+3060->U+305F, U+3062->U+3061, U+3064->U+3063, U+3065->U+3063, U+3067->U+3066, U+3069->U+3068, U+3070->U+306F, U+3071->U+306F, U+3073->U+3072, U+3074->U+3072, U+3076->U+3075, \ 607 | U+3077->U+3075, U+3079->U+3078, U+307A->U+3078, U+307C->U+307B, U+307D->U+307B, U+3084->U+3083, U+3086->U+3085, U+3088->U+3087, U+308F->U+308E, U+3094->U+3046, U+3095->U+304B, U+3096->U+3051, U+30A2->U+30A1, U+30A4->U+30A3, U+30A6->U+30A5, U+30A8->U+30A7, \ 608 | U+30AA->U+30A9, U+30AC->U+30AB, U+30AE->U+30AD, U+30B0->U+30AF, U+30B2->U+30B1, U+30B4->U+30B3, U+30B6->U+30B5, U+30B8->U+30B7, U+30BA->U+30B9, U+30BC->U+30BB, U+30BE->U+30BD, U+30C0->U+30BF, U+30C2->U+30C1, U+30C5->U+30C4, U+30C7->U+30C6, U+30C9->U+30C8, \ 609 | U+30D0->U+30CF, U+30D1->U+30CF, U+30D3->U+30D2, U+30D4->U+30D2, U+30D6->U+30D5, U+30D7->U+30D5, U+30D9->U+30D8, U+30DA->U+30D8, U+30DC->U+30DB, U+30DD->U+30DB, U+30E4->U+30E3, U+30E6->U+30E5, U+30E8->U+30E7, U+30EF->U+30EE, U+30F4->U+30A6, U+30AB->U+30F5, \ 610 | U+30B1->U+30F6, U+30F7->U+30EF, U+30F8->U+30F0, U+30F9->U+30F1, U+30FA->U+30F2, U+30AF->U+31F0, U+30B7->U+31F1, U+30B9->U+31F2, U+30C8->U+31F3, U+30CC->U+31F4, U+30CF->U+31F5, U+30D2->U+31F6, U+30D5->U+31F7, U+30D8->U+31F8, U+30DB->U+31F9, U+30E0->U+31FA, \ 611 | U+30E9->U+31FB, U+30EA->U+31FC, U+30EB->U+31FD, U+30EC->U+31FE, U+30ED->U+31FF, U+FF66->U+30F2, U+FF67->U+30A1, U+FF68->U+30A3, U+FF69->U+30A5, U+FF6A->U+30A7, U+FF6B->U+30A9, U+FF6C->U+30E3, U+FF6D->U+30E5, U+FF6E->U+30E7, U+FF6F->U+30C3, U+FF71->U+30A1, \ 612 | U+FF72->U+30A3, U+FF73->U+30A5, U+FF74->U+30A7, U+FF75->U+30A9, U+FF76->U+30AB, U+FF77->U+30AD, U+FF78->U+30AF, U+FF79->U+30B1, U+FF7A->U+30B3, U+FF7B->U+30B5, U+FF7C->U+30B7, U+FF7D->U+30B9, U+FF7E->U+30BB, U+FF7F->U+30BD, U+FF80->U+30BF, U+FF81->U+30C1, \ 613 | U+FF82->U+30C3, U+FF83->U+30C6, U+FF84->U+30C8, U+FF85->U+30CA, U+FF86->U+30CB, U+FF87->U+30CC, U+FF88->U+30CD, U+FF89->U+30CE, U+FF8A->U+30CF, U+FF8B->U+30D2, U+FF8C->U+30D5, U+FF8D->U+30D8, U+FF8E->U+30DB, U+FF8F->U+30DE, U+FF90->U+30DF, U+FF91->U+30E0, \ 614 | U+FF92->U+30E1, U+FF93->U+30E2, U+FF94->U+30E3, U+FF95->U+30E5, U+FF96->U+30E7, U+FF97->U+30E9, U+FF98->U+30EA, U+FF99->U+30EB, U+FF9A->U+30EC, U+FF9B->U+30ED, U+FF9C->U+30EF, U+FF9D->U+30F3, U+FFA0->U+3164, U+FFA1->U+3131, U+FFA2->U+3132, U+FFA3->U+3133, \ 615 | U+FFA4->U+3134, U+FFA5->U+3135, U+FFA6->U+3136, U+FFA7->U+3137, U+FFA8->U+3138, U+FFA9->U+3139, U+FFAA->U+313A, U+FFAB->U+313B, U+FFAC->U+313C, U+FFAD->U+313D, U+FFAE->U+313E, U+FFAF->U+313F, U+FFB0->U+3140, U+FFB1->U+3141, U+FFB2->U+3142, U+FFB3->U+3143, \ 616 | U+FFB4->U+3144, U+FFB5->U+3145, U+FFB6->U+3146, U+FFB7->U+3147, U+FFB8->U+3148, U+FFB9->U+3149, U+FFBA->U+314A, U+FFBB->U+314B, U+FFBC->U+314C, U+FFBD->U+314D, U+FFBE->U+314E, U+FFC2->U+314F, U+FFC3->U+3150, U+FFC4->U+3151, U+FFC5->U+3152, U+FFC6->U+3153, \ 617 | U+FFC7->U+3154, U+FFCA->U+3155, U+FFCB->U+3156, U+FFCC->U+3157, U+FFCD->U+3158, U+FFCE->U+3159, U+FFCF->U+315A, U+FFD2->U+315B, U+FFD3->U+315C, U+FFD4->U+315D, U+FFD5->U+315E, U+FFD6->U+315F, U+FFD7->U+3160, U+FFDA->U+3161, U+FFDB->U+3162, U+FFDC->U+3163, \ 618 | U+3131->U+1100, U+3132->U+1101, U+3133->U+11AA, U+3134->U+1102, U+3135->U+11AC, U+3136->U+11AD, U+3137->U+1103, U+3138->U+1104, U+3139->U+1105, U+313A->U+11B0, U+313B->U+11B1, U+313C->U+11B2, U+313D->U+11B3, U+313E->U+11B4, U+313F->U+11B5, U+3140->U+111A, \ 619 | U+3141->U+1106, U+3142->U+1107, U+3143->U+1108, U+3144->U+1121, U+3145->U+1109, U+3146->U+110A, U+3147->U+110B, U+3148->U+110C, U+3149->U+110D, U+314A->U+110E, U+314B->U+110F, U+314C->U+1110, U+314D->U+1111, U+314E->U+1112, U+314F->U+1161, U+3150->U+1162, \ 620 | U+3151->U+1163, U+3152->U+1164, U+3153->U+1165, U+3154->U+1166, U+3155->U+1167, U+3156->U+1168, U+3157->U+1169, U+3158->U+116A, U+3159->U+116B, U+315A->U+116C, U+315B->U+116D, U+315C->U+116E, U+315D->U+116F, U+315E->U+1170, U+315F->U+1171, U+3160->U+1172, \ 621 | U+3161->U+1173, U+3162->U+1174, U+3163->U+1175, U+3165->U+1114, U+3166->U+1115, U+3167->U+11C7, U+3168->U+11C8, U+3169->U+11CC, U+316A->U+11CE, U+316B->U+11D3, U+316C->U+11D7, U+316D->U+11D9, U+316E->U+111C, U+316F->U+11DD, U+3170->U+11DF, U+3171->U+111D, \ 622 | U+3172->U+111E, U+3173->U+1120, U+3174->U+1122, U+3175->U+1123, U+3176->U+1127, U+3177->U+1129, U+3178->U+112B, U+3179->U+112C, U+317A->U+112D, U+317B->U+112E, U+317C->U+112F, U+317D->U+1132, U+317E->U+1136, U+317F->U+1140, U+3180->U+1147, U+3181->U+114C, \ 623 | U+3182->U+11F1, U+3183->U+11F2, U+3184->U+1157, U+3185->U+1158, U+3186->U+1159, U+3187->U+1184, U+3188->U+1185, U+3189->U+1188, U+318A->U+1191, U+318B->U+1192, U+318C->U+1194, U+318D->U+119E, U+318E->U+11A1, U+A490->U+A408, U+A491->U+A1B9, U+4E00..U+9FBB, \ 624 | U+3400..U+4DB5, U+20000..U+2A6D6, U+FA0E, U+FA0F, U+FA11, U+FA13, U+FA14, U+FA1F, U+FA21, U+FA23, U+FA24, U+FA27, U+FA28, U+FA29, U+3105..U+312C, U+31A0..U+31B7, U+3041, U+3043, U+3045, U+3047, U+3049, U+304B, U+304D, U+304F, U+3051, U+3053, U+3055, U+3057, \ 625 | U+3059, U+305B, U+305D, U+305F, U+3061, U+3063, U+3066, U+3068, U+306A..U+306F, U+3072, U+3075, U+3078, U+307B, U+307E..U+3083, U+3085, U+3087, U+3089..U+308E, U+3090..U+3093, U+30A1, U+30A3, U+30A5, U+30A7, U+30A9, U+30AD, U+30AF, U+30B3, U+30B5, U+30BB, \ 626 | U+30BD, U+30BF, U+30C1, U+30C3, U+30C4, U+30C6, U+30CA, U+30CB, U+30CD, U+30CE, U+30DE, U+30DF, U+30E1, U+30E2, U+30E3, U+30E5, U+30E7, U+30EE, U+30F0..U+30F3, U+30F5, U+30F6, U+31F0, U+31F1, U+31F2, U+31F3, U+31F4, U+31F5, U+31F6, U+31F7, U+31F8, U+31F9, \ 627 | U+31FA, U+31FB, U+31FC, U+31FD, U+31FE, U+31FF, U+AC00..U+D7A3, U+1100..U+1159, U+1161..U+11A2, U+11A8..U+11F9, U+A000..U+A48C, U+A492..U+A4C6, U+2C80->U+2C81, U+2C81, U+2C82->U+2C83, U+2C83, U+2C84->U+2C85, U+2C85, U+2C86->U+2C87, U+2C87, U+2C88->U+2C89, \ 628 | U+2C89, U+2C8A->U+2C8B, U+2C8B, U+2C8C->U+2C8D, U+2C8D, U+2C8E->U+2C8F, U+2C8F, U+2C90->U+2C91, U+2C91, U+2C92->U+2C93, U+2C93, U+2C94->U+2C95, U+2C95, U+2C96->U+2C97, U+2C97, U+2C98->U+2C99, U+2C99, U+2C9A->U+2C9B, U+2C9B, U+2C9C->U+2C9D, U+2C9D, U+2C9E->U+2C9F, \ 629 | U+2C9F, U+2CA0->U+2CA1, U+2CA1, U+2CA2->U+2CA3, U+2CA3, U+2CA4->U+2CA5, U+2CA5, U+2CA6->U+2CA7, U+2CA7, U+2CA8->U+2CA9, U+2CA9, U+2CAA->U+2CAB, U+2CAB, U+2CAC->U+2CAD, U+2CAD, U+2CAE->U+2CAF, U+2CAF, U+2CB0->U+2CB1, U+2CB1, U+2CB2->U+2CB3, U+2CB3, U+2CB4->U+2CB5, \ 630 | U+2CB5, U+2CB6->U+2CB7, U+2CB7, U+2CB8->U+2CB9, U+2CB9, U+2CBA->U+2CBB, U+2CBB, U+2CBC->U+2CBD, U+2CBD, U+2CBE->U+2CBF, U+2CBF, U+2CC0->U+2CC1, U+2CC1, U+2CC2->U+2CC3, U+2CC3, U+2CC4->U+2CC5, U+2CC5, U+2CC6->U+2CC7, U+2CC7, U+2CC8->U+2CC9, U+2CC9, U+2CCA->U+2CCB, \ 631 | U+2CCB, U+2CCC->U+2CCD, U+2CCD, U+2CCE->U+2CCF, U+2CCF, U+2CD0->U+2CD1, U+2CD1, U+2CD2->U+2CD3, U+2CD3, U+2CD4->U+2CD5, U+2CD5, U+2CD6->U+2CD7, U+2CD7, U+2CD8->U+2CD9, U+2CD9, U+2CDA->U+2CDB, U+2CDB, U+2CDC->U+2CDD, U+2CDD, U+2CDE->U+2CDF, U+2CDF, U+2CE0->U+2CE1, \ 632 | U+2CE1, U+2CE2->U+2CE3, U+2CE3, U+0400->U+0435, U+0401->U+0435, U+0402->U+0452, U+0452, U+0403->U+0433, U+0404->U+0454, U+0454, U+0405->U+0455, U+0455, U+0406->U+0456, U+0407->U+0456, U+0457->U+0456, U+0456, U+0408..U+040B->U+0458..U+045B, U+0458..U+045B, \ 633 | U+040C->U+043A, U+040D->U+0438, U+040E->U+0443, U+040F->U+045F, U+045F, U+0450->U+0435, U+0451->U+0435, U+0453->U+0433, U+045C->U+043A, U+045D->U+0438, U+045E->U+0443, U+0460->U+0461, U+0461, U+0462->U+0463, U+0463, U+0464->U+0465, U+0465, U+0466->U+0467, \ 634 | U+0467, U+0468->U+0469, U+0469, U+046A->U+046B, U+046B, U+046C->U+046D, U+046D, U+046E->U+046F, U+046F, U+0470->U+0471, U+0471, U+0472->U+0473, U+0473, U+0474->U+0475, U+0476->U+0475, U+0477->U+0475, U+0475, U+0478->U+0479, U+0479, U+047A->U+047B, U+047B, \ 635 | U+047C->U+047D, U+047D, U+047E->U+047F, U+047F, U+0480->U+0481, U+0481, U+048A->U+0438, U+048B->U+0438, U+048C->U+044C, U+048D->U+044C, U+048E->U+0440, U+048F->U+0440, U+0490->U+0433, U+0491->U+0433, U+0490->U+0433, U+0491->U+0433, U+0492->U+0433, U+0493->U+0433, \ 636 | U+0494->U+0433, U+0495->U+0433, U+0496->U+0436, U+0497->U+0436, U+0498->U+0437, U+0499->U+0437, U+049A->U+043A, U+049B->U+043A, U+049C->U+043A, U+049D->U+043A, U+049E->U+043A, U+049F->U+043A, U+04A0->U+043A, U+04A1->U+043A, U+04A2->U+043D, U+04A3->U+043D, \ 637 | U+04A4->U+043D, U+04A5->U+043D, U+04A6->U+043F, U+04A7->U+043F, U+04A8->U+04A9, U+04A9, U+04AA->U+0441, U+04AB->U+0441, U+04AC->U+0442, U+04AD->U+0442, U+04AE->U+0443, U+04AF->U+0443, U+04B0->U+0443, U+04B1->U+0443, U+04B2->U+0445, U+04B3->U+0445, U+04B4->U+04B5, \ 638 | U+04B5, U+04B6->U+0447, U+04B7->U+0447, U+04B8->U+0447, U+04B9->U+0447, U+04BA->U+04BB, U+04BB, U+04BC->U+04BD, U+04BE->U+04BD, U+04BF->U+04BD, U+04BD, U+04C0->U+04CF, U+04CF, U+04C1->U+0436, U+04C2->U+0436, U+04C3->U+043A, U+04C4->U+043A, U+04C5->U+043B, \ 639 | U+04C6->U+043B, U+04C7->U+043D, U+04C8->U+043D, U+04C9->U+043D, U+04CA->U+043D, U+04CB->U+0447, U+04CC->U+0447, U+04CD->U+043C, U+04CE->U+043C, U+04D0->U+0430, U+04D1->U+0430, U+04D2->U+0430, U+04D3->U+0430, U+04D4->U+00E6, U+04D5->U+00E6, U+04D6->U+0435, \ 640 | U+04D7->U+0435, U+04D8->U+04D9, U+04DA->U+04D9, U+04DB->U+04D9, U+04D9, U+04DC->U+0436, U+04DD->U+0436, U+04DE->U+0437, U+04DF->U+0437, U+04E0->U+04E1, U+04E1, U+04E2->U+0438, U+04E3->U+0438, U+04E4->U+0438, U+04E5->U+0438, U+04E6->U+043E, U+04E7->U+043E, \ 641 | U+04E8->U+043E, U+04E9->U+043E, U+04EA->U+043E, U+04EB->U+043E, U+04EC->U+044D, U+04ED->U+044D, U+04EE->U+0443, U+04EF->U+0443, U+04F0->U+0443, U+04F1->U+0443, U+04F2->U+0443, U+04F3->U+0443, U+04F4->U+0447, U+04F5->U+0447, U+04F6->U+0433, U+04F7->U+0433, \ 642 | U+04F8->U+044B, U+04F9->U+044B, U+04FA->U+0433, U+04FB->U+0433, U+04FC->U+0445, U+04FD->U+0445, U+04FE->U+0445, U+04FF->U+0445, U+0410..U+0418->U+0430..U+0438, U+0419->U+0438, U+0430..U+0438, U+041A..U+042F->U+043A..U+044F, U+043A..U+044F, U+0929->U+0928, \ 643 | U+0931->U+0930, U+0934->U+0933, U+0958->U+0915, U+0959->U+0916, U+095A->U+0917, U+095B->U+091C, U+095C->U+0921, U+095D->U+0922, U+095E->U+092B, U+095F->U+092F, U+0904..U+0928, U+092A..U+0930, U+0932, U+0933, U+0935..U+0939, U+0960, U+0961, U+0966..U+096F, \ 644 | U+097B..U+097F, U+10FC->U+10DC, U+10D0..U+10FA, U+10A0..U+10C5->U+2D00..U+2D25, U+2D00..U+2D25, U+0386->U+03B1, U+0388->U+03B5, U+0389->U+03B7, U+038A->U+03B9, U+038C->U+03BF, U+038E->U+03C5, U+038F->U+03C9, U+0390->U+03B9, U+03AA->U+03B9, U+03AB->U+03C5, \ 645 | U+03AC->U+03B1, U+03AD->U+03B5, U+03AE->U+03B7, U+03AF->U+03B9, U+03B0->U+03C5, U+03CA->U+03B9, U+03CB->U+03C5, U+03CC->U+03BF, U+03CD->U+03C5, U+03CE->U+03C9, U+03D0->U+03B2, U+03D1->U+03B8, U+03D2->U+03C5, U+03D3->U+03C5, U+03D4->U+03C5, U+03D5->U+03C6, \ 646 | U+03D6->U+03C0, U+03D8->U+03D9, U+03DA->U+03DB, U+03DC->U+03DD, U+03DE->U+03DF, U+03E0->U+03E1, U+03E2->U+03E3, U+03E4->U+03E5, U+03E6->U+03E7, U+03E8->U+03E9, U+03EA->U+03EB, U+03EC->U+03ED, U+03EE->U+03EF, U+03F0->U+03BA, U+03F1->U+03C1, U+03F2->U+03C3, \ 647 | U+03F4->U+03B8, U+03F5->U+03B5, U+03F6->U+03B5, U+03F7->U+03F8, U+03F9->U+03C3, U+03FA->U+03FB, U+1F00->U+03B1, U+1F01->U+03B1, U+1F02->U+03B1, U+1F03->U+03B1, U+1F04->U+03B1, U+1F05->U+03B1, U+1F06->U+03B1, U+1F07->U+03B1, U+1F08->U+03B1, U+1F09->U+03B1, \ 648 | U+1F0A->U+03B1, U+1F0B->U+03B1, U+1F0C->U+03B1, U+1F0D->U+03B1, U+1F0E->U+03B1, U+1F0F->U+03B1, U+1F10->U+03B5, U+1F11->U+03B5, U+1F12->U+03B5, U+1F13->U+03B5, U+1F14->U+03B5, U+1F15->U+03B5, U+1F18->U+03B5, U+1F19->U+03B5, U+1F1A->U+03B5, U+1F1B->U+03B5, \ 649 | U+1F1C->U+03B5, U+1F1D->U+03B5, U+1F20->U+03B7, U+1F21->U+03B7, U+1F22->U+03B7, U+1F23->U+03B7, U+1F24->U+03B7, U+1F25->U+03B7, U+1F26->U+03B7, U+1F27->U+03B7, U+1F28->U+03B7, U+1F29->U+03B7, U+1F2A->U+03B7, U+1F2B->U+03B7, U+1F2C->U+03B7, U+1F2D->U+03B7, \ 650 | U+1F2E->U+03B7, U+1F2F->U+03B7, U+1F30->U+03B9, U+1F31->U+03B9, U+1F32->U+03B9, U+1F33->U+03B9, U+1F34->U+03B9, U+1F35->U+03B9, U+1F36->U+03B9, U+1F37->U+03B9, U+1F38->U+03B9, U+1F39->U+03B9, U+1F3A->U+03B9, U+1F3B->U+03B9, U+1F3C->U+03B9, U+1F3D->U+03B9, \ 651 | U+1F3E->U+03B9, U+1F3F->U+03B9, U+1F40->U+03BF, U+1F41->U+03BF, U+1F42->U+03BF, U+1F43->U+03BF, U+1F44->U+03BF, U+1F45->U+03BF, U+1F48->U+03BF, U+1F49->U+03BF, U+1F4A->U+03BF, U+1F4B->U+03BF, U+1F4C->U+03BF, U+1F4D->U+03BF, U+1F50->U+03C5, U+1F51->U+03C5, \ 652 | U+1F52->U+03C5, U+1F53->U+03C5, U+1F54->U+03C5, U+1F55->U+03C5, U+1F56->U+03C5, U+1F57->U+03C5, U+1F59->U+03C5, U+1F5B->U+03C5, U+1F5D->U+03C5, U+1F5F->U+03C5, U+1F60->U+03C9, U+1F61->U+03C9, U+1F62->U+03C9, U+1F63->U+03C9, U+1F64->U+03C9, U+1F65->U+03C9, \ 653 | U+1F66->U+03C9, U+1F67->U+03C9, U+1F68->U+03C9, U+1F69->U+03C9, U+1F6A->U+03C9, U+1F6B->U+03C9, U+1F6C->U+03C9, U+1F6D->U+03C9, U+1F6E->U+03C9, U+1F6F->U+03C9, U+1F70->U+03B1, U+1F71->U+03B1, U+1F72->U+03B5, U+1F73->U+03B5, U+1F74->U+03B7, U+1F75->U+03B7, \ 654 | U+1F76->U+03B9, U+1F77->U+03B9, U+1F78->U+03BF, U+1F79->U+03BF, U+1F7A->U+03C5, U+1F7B->U+03C5, U+1F7C->U+03C9, U+1F7D->U+03C9, U+1F80->U+03B1, U+1F81->U+03B1, U+1F82->U+03B1, U+1F83->U+03B1, U+1F84->U+03B1, U+1F85->U+03B1, U+1F86->U+03B1, U+1F87->U+03B1, \ 655 | U+1F88->U+03B1, U+1F89->U+03B1, U+1F8A->U+03B1, U+1F8B->U+03B1, U+1F8C->U+03B1, U+1F8D->U+03B1, U+1F8E->U+03B1, U+1F8F->U+03B1, U+1F90->U+03B7, U+1F91->U+03B7, U+1F92->U+03B7, U+1F93->U+03B7, U+1F94->U+03B7, U+1F95->U+03B7, U+1F96->U+03B7, U+1F97->U+03B7, \ 656 | U+1F98->U+03B7, U+1F99->U+03B7, U+1F9A->U+03B7, U+1F9B->U+03B7, U+1F9C->U+03B7, U+1F9D->U+03B7, U+1F9E->U+03B7, U+1F9F->U+03B7, U+1FA0->U+03C9, U+1FA1->U+03C9, U+1FA2->U+03C9, U+1FA3->U+03C9, U+1FA4->U+03C9, U+1FA5->U+03C9, U+1FA6->U+03C9, U+1FA7->U+03C9, \ 657 | U+1FA8->U+03C9, U+1FA9->U+03C9, U+1FAA->U+03C9, U+1FAB->U+03C9, U+1FAC->U+03C9, U+1FAD->U+03C9, U+1FAE->U+03C9, U+1FAF->U+03C9, U+1FB0->U+03B1, U+1FB1->U+03B1, U+1FB2->U+03B1, U+1FB3->U+03B1, U+1FB4->U+03B1, U+1FB6->U+03B1, U+1FB7->U+03B1, U+1FB8->U+03B1, \ 658 | U+1FB9->U+03B1, U+1FBA->U+03B1, U+1FBB->U+03B1, U+1FBC->U+03B1, U+1FC2->U+03B7, U+1FC3->U+03B7, U+1FC4->U+03B7, U+1FC6->U+03B7, U+1FC7->U+03B7, U+1FC8->U+03B5, U+1FC9->U+03B5, U+1FCA->U+03B7, U+1FCB->U+03B7, U+1FCC->U+03B7, U+1FD0->U+03B9, U+1FD1->U+03B9, \ 659 | U+1FD2->U+03B9, U+1FD3->U+03B9, U+1FD6->U+03B9, U+1FD7->U+03B9, U+1FD8->U+03B9, U+1FD9->U+03B9, U+1FDA->U+03B9, U+1FDB->U+03B9, U+1FE0->U+03C5, U+1FE1->U+03C5, U+1FE2->U+03C5, U+1FE3->U+03C5, U+1FE4->U+03C1, U+1FE5->U+03C1, U+1FE6->U+03C5, U+1FE7->U+03C5, \ 660 | U+1FE8->U+03C5, U+1FE9->U+03C5, U+1FEA->U+03C5, U+1FEB->U+03C5, U+1FEC->U+03C1, U+1FF2->U+03C9, U+1FF3->U+03C9, U+1FF4->U+03C9, U+1FF6->U+03C9, U+1FF7->U+03C9, U+1FF8->U+03BF, U+1FF9->U+03BF, U+1FFA->U+03C9, U+1FFB->U+03C9, U+1FFC->U+03C9, U+0391..U+03A1->U+03B1..U+03C1, \ 661 | U+03B1..U+03C1, U+03A3..U+03A9->U+03C3..U+03C9, U+03C3..U+03C9, U+03C2, U+03D9, U+03DB, U+03DD, U+03DF, U+03E1, U+03E3, U+03E5, U+03E7, U+03E9, U+03EB, U+03ED, U+03EF, U+03F3, U+03F8, U+03FB, U+0A85..U+0A8C, U+0A8F, U+0A90, U+0A93..U+0AB0, U+0AB2, U+0AB3, \ 662 | U+0AB5..U+0AB9, U+0AE0, U+0AE1, U+0AE6..U+0AEF, U+0A33->U+0A32, U+0A36->U+0A38, U+0A59->U+0A16, U+0A5A->U+0A17, U+0A5B->U+0A1C, U+0A5E->U+0A2B, U+0A05..U+0A0A, U+0A0F, U+0A10, U+0A13..U+0A28, U+0A2A..U+0A30, U+0A32, U+0A35, U+0A38, U+0A39, U+0A5C, U+0A66..U+0A6F, \ 663 | U+FB1D->U+05D9, U+FB1F->U+05F2, U+FB20->U+05E2, U+FB21->U+05D0, U+FB22->U+05D3, U+FB23->U+05D4, U+FB24->U+05DB, U+FB25->U+05DC, U+FB26->U+05DD, U+FB27->U+05E8, U+FB28->U+05EA, U+FB2A->U+05E9, U+FB2B->U+05E9, U+FB2C->U+05E9, U+FB2D->U+05E9, U+FB2E->U+05D0, \ 664 | U+FB2F->U+05D0, U+FB30->U+05D0, U+FB31->U+05D1, U+FB32->U+05D2, U+FB33->U+05D3, U+FB34->U+05D4, U+FB35->U+05D5, U+FB36->U+05D6, U+FB38->U+05D8, U+FB39->U+05D9, U+FB3A->U+05DA, U+FB3B->U+05DB, U+FB3C->U+05DC, U+FB3E->U+05DE, U+FB40->U+05E0, U+FB41->U+05E1, \ 665 | U+FB43->U+05E3, U+FB44->U+05E4, U+FB46->U+05E6, U+FB47->U+05E7, U+FB48->U+05E8, U+FB49->U+05E9, U+FB4A->U+05EA, U+FB4B->U+05D5, U+FB4C->U+05D1, U+FB4D->U+05DB, U+FB4E->U+05E4, U+FB4F->U+05D0, U+05D0..U+05F2, U+0C85..U+0C8C, U+0C8E..U+0C90, U+0C92..U+0CA8, \ 666 | U+0CAA..U+0CB3, U+0CB5..U+0CB9, U+0CE0, U+0CE1, U+0CE6..U+0CEF, U+1900..U+191C, U+1930..U+1938, U+1946..U+194F, U+0D05..U+0D0C, U+0D0E..U+0D10, U+0D12..U+0D28, U+0D2A..U+0D39, U+0D60, U+0D61, U+0D66..U+0D6F, U+0B94->U+0B92, U+0B85..U+0B8A, U+0B8E..U+0B90, \ 667 | U+0B92, U+0B93, U+0B95, U+0B99, U+0B9A, U+0B9C, U+0B9E, U+0B9F, U+0BA3, U+0BA4, U+0BA8..U+0BAA, U+0BAE..U+0BB9, U+0BE6..U+0BEF, U+0E01..U+0E30, U+0E32, U+0E33, U+0E40..U+0E46, U+0E50..U+0E5B, U+FF10..U+FF19->0..9, U+FF21..U+FF3A->a..z, U+FF41..U+FF5A->a..z 668 | } 669 | 670 | 671 | #inherited index example 672 | # 673 | #all the parameters are copied from the parent index, 674 | # and may then be overridden in this index definition 675 | index haltesstemmed : haltes 676 | { 677 | enable_star = 1 678 | path = /var/ovapi/haltes/haltesstemmed 679 | morphology = libstemmer_dutch 680 | } 681 | 682 | index haltes_metaphone : haltes 683 | { 684 | source = haltes 685 | path = /var/ovapi/haltes/haltesmetaphone 686 | docinfo = extern 687 | enable_star = 1 688 | charset_type = utf-8 689 | ignore_chars = - 690 | morphology = metaphone 691 | } 692 | 693 | # distributed index example 694 | # 695 | # this is a virtual index which can NOT be directly indexed, 696 | # and only contains references to other local and/or remote indexes 697 | index dist1 698 | { 699 | # 'distributed' index type MUST be specified 700 | type = distributed 701 | 702 | # local index to be searched 703 | # there can be many local indexes configured 704 | local = haltes 705 | local = haltesstemmed 706 | 707 | # remote agent 708 | # multiple remote agents may be specified 709 | # syntax for TCP connections is 'hostname:port:index1,[index2[,...]]' 710 | # syntax for local UNIX connections is '/path/to/socket:index1,[index2[,...]]' 711 | agent = localhost:9313:remote1 712 | agent = localhost:9314:remote2,remote3 713 | # agent = /var/run/searchd.sock:remote4 714 | 715 | # blackhole remote agent, for debugging/testing 716 | # network errors and search results will be ignored 717 | # 718 | # agent_blackhole = testbox:9312:testindex1,testindex2 719 | 720 | 721 | # remote agent connection timeout, milliseconds 722 | # optional, default is 1000 ms, ie. 1 sec 723 | agent_connect_timeout = 1000 724 | 725 | # remote agent query timeout, milliseconds 726 | # optional, default is 3000 ms, ie. 3 sec 727 | agent_query_timeout = 3000 728 | } 729 | 730 | ############################################################################# 731 | ## indexer settings 732 | ############################################################################# 733 | 734 | indexer 735 | { 736 | # memory limit, in bytes, kiloytes (16384K) or megabytes (256M) 737 | # optional, default is 32M, max is 2047M, recommended is 256M to 1024M 738 | mem_limit = 32M 739 | 740 | # maximum IO calls per second (for I/O throttling) 741 | # optional, default is 0 (unlimited) 742 | # 743 | # max_iops = 40 744 | 745 | 746 | # maximum IO call size, bytes (for I/O throttling) 747 | # optional, default is 0 (unlimited) 748 | # 749 | # max_iosize = 1048576 750 | 751 | 752 | # maximum xmlpipe2 field length, bytes 753 | # optional, default is 2M 754 | # 755 | # max_xmlpipe2_field = 4M 756 | 757 | 758 | # write buffer size, bytes 759 | # several (currently up to 4) buffers will be allocated 760 | # write buffers are allocated in addition to mem_limit 761 | # optional, default is 1M 762 | # 763 | # write_buffer = 1M 764 | } 765 | 766 | ############################################################################# 767 | ## searchd settings 768 | ############################################################################# 769 | 770 | searchd 771 | { 772 | # hostname, port, or hostname:port, or /unix/socket/path to listen on 773 | # multi-value, multiple listen points are allowed 774 | # optional, default is 0.0.0.0:9312 (listen on all interfaces, port 9312) 775 | # 776 | #listen = 127.0.0.1 777 | #listen = 192.168.0.1:9312 778 | #listen = 9312 779 | # listen = /var/run/searchd.sock 780 | 781 | 782 | # log file, searchd run info is logged here 783 | # optional, default is 'searchd.log' 784 | log = /var/log/sphinxsearch/searchd.log 785 | 786 | # query log file, all search queries are logged here 787 | # optional, default is empty (do not log queries) 788 | query_log = /var/log/sphinxsearch/query.log 789 | 790 | # client read timeout, seconds 791 | # optional, default is 5 792 | read_timeout = 5 793 | 794 | # request timeout, seconds 795 | # optional, default is 5 minutes 796 | client_timeout = 300 797 | 798 | # maximum amount of children to fork (concurrent searches to run) 799 | # optional, default is 0 (unlimited) 800 | max_children = 30 801 | 802 | # PID file, searchd process ID file name 803 | # mandatory 804 | pid_file = /var/ovapi/haltes/searchd.pid 805 | 806 | # max amount of matches the daemon ever keeps in RAM, per-index 807 | # WARNING, THERE'S ALSO PER-QUERY LIMIT, SEE SetLimits() API CALL 808 | # default is 1000 (just like Google) 809 | max_matches = 1000 810 | 811 | # seamless rotate, prevents rotate stalls if precaching huge datasets 812 | # optional, default is 1 813 | seamless_rotate = 1 814 | 815 | # whether to forcibly preopen all indexes on startup 816 | # optional, default is 0 (do not preopen) 817 | preopen_indexes = 0 818 | 819 | # whether to unlink .old index copies on succesful rotation. 820 | # optional, default is 1 (do unlink) 821 | unlink_old = 1 822 | 823 | # attribute updates periodic flush timeout, seconds 824 | # updates will be automatically dumped to disk this frequently 825 | # optional, default is 0 (disable periodic flush) 826 | # 827 | # attr_flush_period = 900 828 | 829 | 830 | # instance-wide ondisk_dict defaults (per-index value take precedence) 831 | # optional, default is 0 (precache all dictionaries in RAM) 832 | # 833 | # ondisk_dict_default = 1 834 | 835 | 836 | # MVA updates pool size 837 | # shared between all instances of searchd, disables attr flushes! 838 | # optional, default size is 1M 839 | mva_updates_pool = 1M 840 | 841 | # max allowed network packet size 842 | # limits both query packets from clients, and responses from agents 843 | # optional, default size is 8M 844 | max_packet_size = 8M 845 | 846 | # crash log path 847 | # searchd will (try to) log crashed query to 'crash_log_path.PID' file 848 | # optional, default is empty (do not create crash logs) 849 | # 850 | # crash_log_path = /var/log/sphinxsearch/crash 851 | 852 | 853 | # max allowed per-query filter count 854 | # optional, default is 256 855 | max_filters = 256 856 | 857 | # max allowed per-filter values count 858 | # optional, default is 4096 859 | max_filter_values = 4096 860 | 861 | 862 | # socket listen queue length 863 | # optional, default is 5 864 | # 865 | # listen_backlog = 5 866 | 867 | 868 | # per-keyword read buffer size 869 | # optional, default is 256K 870 | # 871 | # read_buffer = 256K 872 | 873 | 874 | # unhinted read size (currently used when reading hits) 875 | # optional, default is 32K 876 | # 877 | # read_unhinted = 32K 878 | } 879 | 880 | # --eof-- 881 | 882 | -------------------------------------------------------------------------------- /sphinxapi.py: -------------------------------------------------------------------------------- 1 | # 2 | # $Id: sphinxapi.py 3087 2012-01-30 23:07:35Z shodan $ 3 | # 4 | # Python version of Sphinx searchd client (Python API) 5 | # 6 | # Copyright (c) 2006, Mike Osadnik 7 | # Copyright (c) 2006-2012, Andrew Aksyonoff 8 | # Copyright (c) 2008-2012, Sphinx Technologies Inc 9 | # All rights reserved 10 | # 11 | # This program is free software; you can redistribute it and/or modify 12 | # it under the terms of the GNU General Public License. You should have 13 | # received a copy of the GPL license along with this program; if you 14 | # did not, you can find it at http://www.gnu.org/ 15 | # 16 | 17 | import sys 18 | import select 19 | import socket 20 | import re 21 | from struct import * 22 | 23 | 24 | # known searchd commands 25 | SEARCHD_COMMAND_SEARCH = 0 26 | SEARCHD_COMMAND_EXCERPT = 1 27 | SEARCHD_COMMAND_UPDATE = 2 28 | SEARCHD_COMMAND_KEYWORDS = 3 29 | SEARCHD_COMMAND_PERSIST = 4 30 | SEARCHD_COMMAND_STATUS = 5 31 | SEARCHD_COMMAND_FLUSHATTRS = 7 32 | 33 | # current client-side command implementation versions 34 | VER_COMMAND_SEARCH = 0x119 35 | VER_COMMAND_EXCERPT = 0x104 36 | VER_COMMAND_UPDATE = 0x102 37 | VER_COMMAND_KEYWORDS = 0x100 38 | VER_COMMAND_STATUS = 0x100 39 | VER_COMMAND_FLUSHATTRS = 0x100 40 | 41 | # known searchd status codes 42 | SEARCHD_OK = 0 43 | SEARCHD_ERROR = 1 44 | SEARCHD_RETRY = 2 45 | SEARCHD_WARNING = 3 46 | 47 | # known match modes 48 | SPH_MATCH_ALL = 0 49 | SPH_MATCH_ANY = 1 50 | SPH_MATCH_PHRASE = 2 51 | SPH_MATCH_BOOLEAN = 3 52 | SPH_MATCH_EXTENDED = 4 53 | SPH_MATCH_FULLSCAN = 5 54 | SPH_MATCH_EXTENDED2 = 6 55 | 56 | # known ranking modes (extended2 mode only) 57 | SPH_RANK_PROXIMITY_BM25 = 0 # default mode, phrase proximity major factor and BM25 minor one 58 | SPH_RANK_BM25 = 1 # statistical mode, BM25 ranking only (faster but worse quality) 59 | SPH_RANK_NONE = 2 # no ranking, all matches get a weight of 1 60 | SPH_RANK_WORDCOUNT = 3 # simple word-count weighting, rank is a weighted sum of per-field keyword occurence counts 61 | SPH_RANK_PROXIMITY = 4 62 | SPH_RANK_MATCHANY = 5 63 | SPH_RANK_FIELDMASK = 6 64 | SPH_RANK_SPH04 = 7 65 | SPH_RANK_EXPR = 8 66 | SPH_RANK_TOTAL = 9 67 | 68 | # known sort modes 69 | SPH_SORT_RELEVANCE = 0 70 | SPH_SORT_ATTR_DESC = 1 71 | SPH_SORT_ATTR_ASC = 2 72 | SPH_SORT_TIME_SEGMENTS = 3 73 | SPH_SORT_EXTENDED = 4 74 | SPH_SORT_EXPR = 5 75 | 76 | # known filter types 77 | SPH_FILTER_VALUES = 0 78 | SPH_FILTER_RANGE = 1 79 | SPH_FILTER_FLOATRANGE = 2 80 | 81 | # known attribute types 82 | SPH_ATTR_NONE = 0 83 | SPH_ATTR_INTEGER = 1 84 | SPH_ATTR_TIMESTAMP = 2 85 | SPH_ATTR_ORDINAL = 3 86 | SPH_ATTR_BOOL = 4 87 | SPH_ATTR_FLOAT = 5 88 | SPH_ATTR_BIGINT = 6 89 | SPH_ATTR_STRING = 7 90 | SPH_ATTR_MULTI = 0X40000001L 91 | SPH_ATTR_MULTI64 = 0X40000002L 92 | 93 | SPH_ATTR_TYPES = (SPH_ATTR_NONE, 94 | SPH_ATTR_INTEGER, 95 | SPH_ATTR_TIMESTAMP, 96 | SPH_ATTR_ORDINAL, 97 | SPH_ATTR_BOOL, 98 | SPH_ATTR_FLOAT, 99 | SPH_ATTR_BIGINT, 100 | SPH_ATTR_STRING, 101 | SPH_ATTR_MULTI, 102 | SPH_ATTR_MULTI64) 103 | 104 | # known grouping functions 105 | SPH_GROUPBY_DAY = 0 106 | SPH_GROUPBY_WEEK = 1 107 | SPH_GROUPBY_MONTH = 2 108 | SPH_GROUPBY_YEAR = 3 109 | SPH_GROUPBY_ATTR = 4 110 | SPH_GROUPBY_ATTRPAIR = 5 111 | 112 | 113 | class SphinxClient: 114 | def __init__ (self): 115 | """ 116 | Create a new client object, and fill defaults. 117 | """ 118 | self._host = 'localhost' # searchd host (default is "localhost") 119 | self._port = 9312 # searchd port (default is 9312) 120 | self._path = None # searchd unix-domain socket path 121 | self._socket = None 122 | self._offset = 0 # how much records to seek from result-set start (default is 0) 123 | self._limit = 20 # how much records to return from result-set starting at offset (default is 20) 124 | self._mode = SPH_MATCH_ALL # query matching mode (default is SPH_MATCH_ALL) 125 | self._weights = [] # per-field weights (default is 1 for all fields) 126 | self._sort = SPH_SORT_RELEVANCE # match sorting mode (default is SPH_SORT_RELEVANCE) 127 | self._sortby = '' # attribute to sort by (defualt is "") 128 | self._min_id = 0 # min ID to match (default is 0) 129 | self._max_id = 0 # max ID to match (default is UINT_MAX) 130 | self._filters = [] # search filters 131 | self._groupby = '' # group-by attribute name 132 | self._groupfunc = SPH_GROUPBY_DAY # group-by function (to pre-process group-by attribute value with) 133 | self._groupsort = '@group desc' # group-by sorting clause (to sort groups in result set with) 134 | self._groupdistinct = '' # group-by count-distinct attribute 135 | self._maxmatches = 1000 # max matches to retrieve 136 | self._cutoff = 0 # cutoff to stop searching at 137 | self._retrycount = 0 # distributed retry count 138 | self._retrydelay = 0 # distributed retry delay 139 | self._anchor = {} # geographical anchor point 140 | self._indexweights = {} # per-index weights 141 | self._ranker = SPH_RANK_PROXIMITY_BM25 # ranking mode 142 | self._rankexpr = '' # ranking expression for SPH_RANK_EXPR 143 | self._maxquerytime = 0 # max query time, milliseconds (default is 0, do not limit) 144 | self._timeout = 1.0 # connection timeout 145 | self._fieldweights = {} # per-field-name weights 146 | self._overrides = {} # per-query attribute values overrides 147 | self._select = '*' # select-list (attributes or expressions, with optional aliases) 148 | 149 | self._error = '' # last error message 150 | self._warning = '' # last warning message 151 | self._reqs = [] # requests array for multi-query 152 | 153 | def __del__ (self): 154 | if self._socket: 155 | self._socket.close() 156 | 157 | 158 | def GetLastError (self): 159 | """ 160 | Get last error message (string). 161 | """ 162 | return self._error 163 | 164 | 165 | def GetLastWarning (self): 166 | """ 167 | Get last warning message (string). 168 | """ 169 | return self._warning 170 | 171 | 172 | def SetServer (self, host, port = None): 173 | """ 174 | Set searchd server host and port. 175 | """ 176 | assert(isinstance(host, str)) 177 | if host.startswith('/'): 178 | self._path = host 179 | return 180 | elif host.startswith('unix://'): 181 | self._path = host[7:] 182 | return 183 | assert(isinstance(port, int)) 184 | self._host = host 185 | self._port = port 186 | self._path = None 187 | 188 | def SetConnectTimeout ( self, timeout ): 189 | """ 190 | Set connection timeout ( float second ) 191 | """ 192 | assert (isinstance(timeout, float)) 193 | # set timeout to 0 make connaection non-blocking that is wrong so timeout got clipped to reasonable minimum 194 | self._timeout = max ( 0.001, timeout ) 195 | 196 | def _Connect (self): 197 | """ 198 | INTERNAL METHOD, DO NOT CALL. Connects to searchd server. 199 | """ 200 | if self._socket: 201 | # we have a socket, but is it still alive? 202 | sr, sw, _ = select.select ( [self._socket], [self._socket], [], 0 ) 203 | 204 | # this is how alive socket should look 205 | if len(sr)==0 and len(sw)==1: 206 | return self._socket 207 | 208 | # oops, looks like it was closed, lets reopen 209 | self._socket.close() 210 | self._socket = None 211 | 212 | try: 213 | if self._path: 214 | af = socket.AF_UNIX 215 | addr = self._path 216 | desc = self._path 217 | else: 218 | af = socket.AF_INET 219 | addr = ( self._host, self._port ) 220 | desc = '%s;%s' % addr 221 | sock = socket.socket ( af, socket.SOCK_STREAM ) 222 | sock.settimeout ( self._timeout ) 223 | sock.connect ( addr ) 224 | except socket.error, msg: 225 | if sock: 226 | sock.close() 227 | self._error = 'connection to %s failed (%s)' % ( desc, msg ) 228 | return 229 | 230 | v = unpack('>L', sock.recv(4)) 231 | if v<1: 232 | sock.close() 233 | self._error = 'expected searchd protocol version, got %s' % v 234 | return 235 | 236 | # all ok, send my version 237 | sock.send(pack('>L', 1)) 238 | return sock 239 | 240 | 241 | def _GetResponse (self, sock, client_ver): 242 | """ 243 | INTERNAL METHOD, DO NOT CALL. Gets and checks response packet from searchd server. 244 | """ 245 | (status, ver, length) = unpack('>2HL', sock.recv(8)) 246 | response = '' 247 | left = length 248 | while left>0: 249 | chunk = sock.recv(left) 250 | if chunk: 251 | response += chunk 252 | left -= len(chunk) 253 | else: 254 | break 255 | 256 | if not self._socket: 257 | sock.close() 258 | 259 | # check response 260 | read = len(response) 261 | if not response or read!=length: 262 | if length: 263 | self._error = 'failed to read searchd response (status=%s, ver=%s, len=%s, read=%s)' \ 264 | % (status, ver, length, read) 265 | else: 266 | self._error = 'received zero-sized searchd response' 267 | return None 268 | 269 | # check status 270 | if status==SEARCHD_WARNING: 271 | wend = 4 + unpack ( '>L', response[0:4] )[0] 272 | self._warning = response[4:wend] 273 | return response[wend:] 274 | 275 | if status==SEARCHD_ERROR: 276 | self._error = 'searchd error: '+response[4:] 277 | return None 278 | 279 | if status==SEARCHD_RETRY: 280 | self._error = 'temporary searchd error: '+response[4:] 281 | return None 282 | 283 | if status!=SEARCHD_OK: 284 | self._error = 'unknown status code %d' % status 285 | return None 286 | 287 | # check version 288 | if ver>8, ver&0xff, client_ver>>8, client_ver&0xff) 291 | 292 | return response 293 | 294 | 295 | def SetLimits (self, offset, limit, maxmatches=0, cutoff=0): 296 | """ 297 | Set offset and count into result set, and optionally set max-matches and cutoff limits. 298 | """ 299 | assert ( type(offset) in [int,long] and 0<=offset<16777216 ) 300 | assert ( type(limit) in [int,long] and 0=0) 302 | self._offset = offset 303 | self._limit = limit 304 | if maxmatches>0: 305 | self._maxmatches = maxmatches 306 | if cutoff>=0: 307 | self._cutoff = cutoff 308 | 309 | 310 | def SetMaxQueryTime (self, maxquerytime): 311 | """ 312 | Set maximum query time, in milliseconds, per-index. 0 means 'do not limit'. 313 | """ 314 | assert(isinstance(maxquerytime,int) and maxquerytime>0) 315 | self._maxquerytime = maxquerytime 316 | 317 | 318 | def SetMatchMode (self, mode): 319 | """ 320 | Set matching mode. 321 | """ 322 | assert(mode in [SPH_MATCH_ALL, SPH_MATCH_ANY, SPH_MATCH_PHRASE, SPH_MATCH_BOOLEAN, SPH_MATCH_EXTENDED, SPH_MATCH_FULLSCAN, SPH_MATCH_EXTENDED2]) 323 | self._mode = mode 324 | 325 | 326 | def SetRankingMode ( self, ranker, rankexpr='' ): 327 | """ 328 | Set ranking mode. 329 | """ 330 | assert(ranker>=0 and ranker=0) 456 | assert(isinstance(delay,int) and delay>=0) 457 | self._retrycount = count 458 | self._retrydelay = delay 459 | 460 | 461 | def SetOverride (self, name, type, values): 462 | assert(isinstance(name, str)) 463 | assert(type in SPH_ATTR_TYPES) 464 | assert(isinstance(values, dict)) 465 | 466 | self._overrides[name] = {'name': name, 'type': type, 'values': values} 467 | 468 | def SetSelect (self, select): 469 | assert(isinstance(select, str)) 470 | self._select = select 471 | 472 | 473 | def ResetOverrides (self): 474 | self._overrides = {} 475 | 476 | 477 | def ResetFilters (self): 478 | """ 479 | Clear all filters (for multi-queries). 480 | """ 481 | self._filters = [] 482 | self._anchor = {} 483 | 484 | 485 | def ResetGroupBy (self): 486 | """ 487 | Clear groupby settings (for multi-queries). 488 | """ 489 | self._groupby = '' 490 | self._groupfunc = SPH_GROUPBY_DAY 491 | self._groupsort = '@group desc' 492 | self._groupdistinct = '' 493 | 494 | 495 | def Query (self, query, index='*', comment=''): 496 | """ 497 | Connect to searchd server and run given search query. 498 | Returns None on failure; result set hash on success (see documentation for details). 499 | """ 500 | assert(len(self._reqs)==0) 501 | self.AddQuery(query,index,comment) 502 | results = self.RunQueries() 503 | self._reqs = [] # we won't re-run erroneous batch 504 | 505 | if not results or len(results)==0: 506 | return None 507 | self._error = results[0]['error'] 508 | self._warning = results[0]['warning'] 509 | if results[0]['status'] == SEARCHD_ERROR: 510 | return None 511 | return results[0] 512 | 513 | 514 | def AddQuery (self, query, index='*', comment=''): 515 | """ 516 | Add query to batch. 517 | """ 518 | # build request 519 | req = [] 520 | req.append(pack('>4L', self._offset, self._limit, self._mode, self._ranker)) 521 | if self._ranker==SPH_RANK_EXPR: 522 | req.append(pack('>L', len(self._rankexpr))) 523 | req.append(self._rankexpr) 524 | req.append(pack('>L', self._sort)) 525 | req.append(pack('>L', len(self._sortby))) 526 | req.append(self._sortby) 527 | 528 | if isinstance(query,unicode): 529 | query = query.encode('utf-8') 530 | assert(isinstance(query,str)) 531 | 532 | req.append(pack('>L', len(query))) 533 | req.append(query) 534 | 535 | req.append(pack('>L', len(self._weights))) 536 | for w in self._weights: 537 | req.append(pack('>L', w)) 538 | req.append(pack('>L', len(index))) 539 | req.append(index) 540 | req.append(pack('>L',1)) # id64 range marker 541 | req.append(pack('>Q', self._min_id)) 542 | req.append(pack('>Q', self._max_id)) 543 | 544 | # filters 545 | req.append ( pack ( '>L', len(self._filters) ) ) 546 | for f in self._filters: 547 | req.append ( pack ( '>L', len(f['attr'])) + f['attr']) 548 | filtertype = f['type'] 549 | req.append ( pack ( '>L', filtertype)) 550 | if filtertype == SPH_FILTER_VALUES: 551 | req.append ( pack ('>L', len(f['values']))) 552 | for val in f['values']: 553 | req.append ( pack ('>q', val)) 554 | elif filtertype == SPH_FILTER_RANGE: 555 | req.append ( pack ('>2q', f['min'], f['max'])) 556 | elif filtertype == SPH_FILTER_FLOATRANGE: 557 | req.append ( pack ('>2f', f['min'], f['max'])) 558 | req.append ( pack ( '>L', f['exclude'] ) ) 559 | 560 | # group-by, max-matches, group-sort 561 | req.append ( pack ( '>2L', self._groupfunc, len(self._groupby) ) ) 562 | req.append ( self._groupby ) 563 | req.append ( pack ( '>2L', self._maxmatches, len(self._groupsort) ) ) 564 | req.append ( self._groupsort ) 565 | req.append ( pack ( '>LLL', self._cutoff, self._retrycount, self._retrydelay)) 566 | req.append ( pack ( '>L', len(self._groupdistinct))) 567 | req.append ( self._groupdistinct) 568 | 569 | # anchor point 570 | if len(self._anchor) == 0: 571 | req.append ( pack ('>L', 0)) 572 | else: 573 | attrlat, attrlong = self._anchor['attrlat'], self._anchor['attrlong'] 574 | latitude, longitude = self._anchor['lat'], self._anchor['long'] 575 | req.append ( pack ('>L', 1)) 576 | req.append ( pack ('>L', len(attrlat)) + attrlat) 577 | req.append ( pack ('>L', len(attrlong)) + attrlong) 578 | req.append ( pack ('>f', latitude) + pack ('>f', longitude)) 579 | 580 | # per-index weights 581 | req.append ( pack ('>L',len(self._indexweights))) 582 | for indx,weight in self._indexweights.items(): 583 | req.append ( pack ('>L',len(indx)) + indx + pack ('>L',weight)) 584 | 585 | # max query time 586 | req.append ( pack ('>L', self._maxquerytime) ) 587 | 588 | # per-field weights 589 | req.append ( pack ('>L',len(self._fieldweights) ) ) 590 | for field,weight in self._fieldweights.items(): 591 | req.append ( pack ('>L',len(field)) + field + pack ('>L',weight) ) 592 | 593 | # comment 594 | req.append ( pack('>L',len(comment)) + comment ) 595 | 596 | # attribute overrides 597 | req.append ( pack('>L', len(self._overrides)) ) 598 | for v in self._overrides.values(): 599 | req.extend ( ( pack('>L', len(v['name'])), v['name'] ) ) 600 | req.append ( pack('>LL', v['type'], len(v['values'])) ) 601 | for id, value in v['values'].iteritems(): 602 | req.append ( pack('>Q', id) ) 603 | if v['type'] == SPH_ATTR_FLOAT: 604 | req.append ( pack('>f', value) ) 605 | elif v['type'] == SPH_ATTR_BIGINT: 606 | req.append ( pack('>q', value) ) 607 | else: 608 | req.append ( pack('>l', value) ) 609 | 610 | # select-list 611 | req.append ( pack('>L', len(self._select)) ) 612 | req.append ( self._select ) 613 | 614 | # send query, get response 615 | req = ''.join(req) 616 | 617 | self._reqs.append(req) 618 | return 619 | 620 | 621 | def RunQueries (self): 622 | """ 623 | Run queries batch. 624 | Returns None on network IO failure; or an array of result set hashes on success. 625 | """ 626 | if len(self._reqs)==0: 627 | self._error = 'no queries defined, issue AddQuery() first' 628 | return None 629 | 630 | sock = self._Connect() 631 | if not sock: 632 | return None 633 | 634 | req = ''.join(self._reqs) 635 | length = len(req)+8 636 | req = pack('>HHLLL', SEARCHD_COMMAND_SEARCH, VER_COMMAND_SEARCH, length, 0, len(self._reqs))+req 637 | sock.send(req) 638 | 639 | response = self._GetResponse(sock, VER_COMMAND_SEARCH) 640 | if not response: 641 | return None 642 | 643 | nreqs = len(self._reqs) 644 | 645 | # parse response 646 | max_ = len(response) 647 | p = 0 648 | 649 | results = [] 650 | for i in range(0,nreqs,1): 651 | result = {} 652 | results.append(result) 653 | 654 | result['error'] = '' 655 | result['warning'] = '' 656 | status = unpack('>L', response[p:p+4])[0] 657 | p += 4 658 | result['status'] = status 659 | if status != SEARCHD_OK: 660 | length = unpack('>L', response[p:p+4])[0] 661 | p += 4 662 | message = response[p:p+length] 663 | p += length 664 | 665 | if status == SEARCHD_WARNING: 666 | result['warning'] = message 667 | else: 668 | result['error'] = message 669 | continue 670 | 671 | # read schema 672 | fields = [] 673 | attrs = [] 674 | 675 | nfields = unpack('>L', response[p:p+4])[0] 676 | p += 4 677 | while nfields>0 and pL', response[p:p+4])[0] 680 | p += 4 681 | fields.append(response[p:p+length]) 682 | p += length 683 | 684 | result['fields'] = fields 685 | 686 | nattrs = unpack('>L', response[p:p+4])[0] 687 | p += 4 688 | while nattrs>0 and pL', response[p:p+4])[0] 691 | p += 4 692 | attr = response[p:p+length] 693 | p += length 694 | type_ = unpack('>L', response[p:p+4])[0] 695 | p += 4 696 | attrs.append([attr,type_]) 697 | 698 | result['attrs'] = attrs 699 | 700 | # read match count 701 | count = unpack('>L', response[p:p+4])[0] 702 | p += 4 703 | id64 = unpack('>L', response[p:p+4])[0] 704 | p += 4 705 | 706 | # read matches 707 | result['matches'] = [] 708 | while count>0 and pQL', response[p:p+12]) 712 | p += 12 713 | else: 714 | doc, weight = unpack('>2L', response[p:p+8]) 715 | p += 8 716 | 717 | match = { 'id':doc, 'weight':weight, 'attrs':{} } 718 | for i in range(len(attrs)): 719 | if attrs[i][1] == SPH_ATTR_FLOAT: 720 | match['attrs'][attrs[i][0]] = unpack('>f', response[p:p+4])[0] 721 | elif attrs[i][1] == SPH_ATTR_BIGINT: 722 | match['attrs'][attrs[i][0]] = unpack('>q', response[p:p+8])[0] 723 | p += 4 724 | elif attrs[i][1] == SPH_ATTR_STRING: 725 | slen = unpack('>L', response[p:p+4])[0] 726 | p += 4 727 | match['attrs'][attrs[i][0]] = '' 728 | if slen>0: 729 | match['attrs'][attrs[i][0]] = response[p:p+slen] 730 | p += slen-4 731 | elif attrs[i][1] == SPH_ATTR_MULTI: 732 | match['attrs'][attrs[i][0]] = [] 733 | nvals = unpack('>L', response[p:p+4])[0] 734 | p += 4 735 | for n in range(0,nvals,1): 736 | match['attrs'][attrs[i][0]].append(unpack('>L', response[p:p+4])[0]) 737 | p += 4 738 | p -= 4 739 | elif attrs[i][1] == SPH_ATTR_MULTI64: 740 | match['attrs'][attrs[i][0]] = [] 741 | nvals = unpack('>L', response[p:p+4])[0] 742 | nvals = nvals/2 743 | p += 4 744 | for n in range(0,nvals,1): 745 | match['attrs'][attrs[i][0]].append(unpack('>q', response[p:p+8])[0]) 746 | p += 8 747 | p -= 4 748 | else: 749 | match['attrs'][attrs[i][0]] = unpack('>L', response[p:p+4])[0] 750 | p += 4 751 | 752 | result['matches'].append ( match ) 753 | 754 | result['total'], result['total_found'], result['time'], words = unpack('>4L', response[p:p+16]) 755 | 756 | result['time'] = '%.3f' % (result['time']/1000.0) 757 | p += 16 758 | 759 | result['words'] = [] 760 | while words>0: 761 | words -= 1 762 | length = unpack('>L', response[p:p+4])[0] 763 | p += 4 764 | word = response[p:p+length] 765 | p += length 766 | docs, hits = unpack('>2L', response[p:p+8]) 767 | p += 8 768 | 769 | result['words'].append({'word':word, 'docs':docs, 'hits':hits}) 770 | 771 | self._reqs = [] 772 | return results 773 | 774 | 775 | def BuildExcerpts (self, docs, index, words, opts=None): 776 | """ 777 | Connect to searchd server and generate exceprts from given documents. 778 | """ 779 | if not opts: 780 | opts = {} 781 | if isinstance(words,unicode): 782 | words = words.encode('utf-8') 783 | 784 | assert(isinstance(docs, list)) 785 | assert(isinstance(index, str)) 786 | assert(isinstance(words, str)) 787 | assert(isinstance(opts, dict)) 788 | 789 | sock = self._Connect() 790 | 791 | if not sock: 792 | return None 793 | 794 | # fixup options 795 | opts.setdefault('before_match', '') 796 | opts.setdefault('after_match', '') 797 | opts.setdefault('chunk_separator', ' ... ') 798 | opts.setdefault('html_strip_mode', 'index') 799 | opts.setdefault('limit', 256) 800 | opts.setdefault('limit_passages', 0) 801 | opts.setdefault('limit_words', 0) 802 | opts.setdefault('around', 5) 803 | opts.setdefault('start_passage_id', 1) 804 | opts.setdefault('passage_boundary', 'none') 805 | 806 | # build request 807 | # v.1.0 req 808 | 809 | flags = 1 # (remove spaces) 810 | if opts.get('exact_phrase'): flags |= 2 811 | if opts.get('single_passage'): flags |= 4 812 | if opts.get('use_boundaries'): flags |= 8 813 | if opts.get('weight_order'): flags |= 16 814 | if opts.get('query_mode'): flags |= 32 815 | if opts.get('force_all_words'): flags |= 64 816 | if opts.get('load_files'): flags |= 128 817 | if opts.get('allow_empty'): flags |= 256 818 | if opts.get('emit_zones'): flags |= 512 819 | if opts.get('load_files_scattered'): flags |= 1024 820 | 821 | # mode=0, flags 822 | req = [pack('>2L', 0, flags)] 823 | 824 | # req index 825 | req.append(pack('>L', len(index))) 826 | req.append(index) 827 | 828 | # req words 829 | req.append(pack('>L', len(words))) 830 | req.append(words) 831 | 832 | # options 833 | req.append(pack('>L', len(opts['before_match']))) 834 | req.append(opts['before_match']) 835 | 836 | req.append(pack('>L', len(opts['after_match']))) 837 | req.append(opts['after_match']) 838 | 839 | req.append(pack('>L', len(opts['chunk_separator']))) 840 | req.append(opts['chunk_separator']) 841 | 842 | req.append(pack('>L', int(opts['limit']))) 843 | req.append(pack('>L', int(opts['around']))) 844 | 845 | req.append(pack('>L', int(opts['limit_passages']))) 846 | req.append(pack('>L', int(opts['limit_words']))) 847 | req.append(pack('>L', int(opts['start_passage_id']))) 848 | req.append(pack('>L', len(opts['html_strip_mode']))) 849 | req.append((opts['html_strip_mode'])) 850 | req.append(pack('>L', len(opts['passage_boundary']))) 851 | req.append((opts['passage_boundary'])) 852 | 853 | # documents 854 | req.append(pack('>L', len(docs))) 855 | for doc in docs: 856 | if isinstance(doc,unicode): 857 | doc = doc.encode('utf-8') 858 | assert(isinstance(doc, str)) 859 | req.append(pack('>L', len(doc))) 860 | req.append(doc) 861 | 862 | req = ''.join(req) 863 | 864 | # send query, get response 865 | length = len(req) 866 | 867 | # add header 868 | req = pack('>2HL', SEARCHD_COMMAND_EXCERPT, VER_COMMAND_EXCERPT, length)+req 869 | wrote = sock.send(req) 870 | 871 | response = self._GetResponse(sock, VER_COMMAND_EXCERPT ) 872 | if not response: 873 | return [] 874 | 875 | # parse response 876 | pos = 0 877 | res = [] 878 | rlen = len(response) 879 | 880 | for i in range(len(docs)): 881 | length = unpack('>L', response[pos:pos+4])[0] 882 | pos += 4 883 | 884 | if pos+length > rlen: 885 | self._error = 'incomplete reply' 886 | return [] 887 | 888 | res.append(response[pos:pos+length]) 889 | pos += length 890 | 891 | return res 892 | 893 | 894 | def UpdateAttributes ( self, index, attrs, values, mva=False ): 895 | """ 896 | Update given attribute values on given documents in given indexes. 897 | Returns amount of updated documents (0 or more) on success, or -1 on failure. 898 | 899 | 'attrs' must be a list of strings. 900 | 'values' must be a dict with int key (document ID) and list of int values (new attribute values). 901 | optional boolean parameter 'mva' points that there is update of MVA attributes. 902 | In this case the 'values' must be a dict with int key (document ID) and list of lists of int values 903 | (new MVA attribute values). 904 | 905 | Example: 906 | res = cl.UpdateAttributes ( 'test1', [ 'group_id', 'date_added' ], { 2:[123,1000000000], 4:[456,1234567890] } ) 907 | """ 908 | assert ( isinstance ( index, str ) ) 909 | assert ( isinstance ( attrs, list ) ) 910 | assert ( isinstance ( values, dict ) ) 911 | for attr in attrs: 912 | assert ( isinstance ( attr, str ) ) 913 | for docid, entry in values.items(): 914 | AssertUInt32(docid) 915 | assert ( isinstance ( entry, list ) ) 916 | assert ( len(attrs)==len(entry) ) 917 | for val in entry: 918 | if mva: 919 | assert ( isinstance ( val, list ) ) 920 | for vals in val: 921 | AssertInt32(vals) 922 | else: 923 | AssertInt32(val) 924 | 925 | # build request 926 | req = [ pack('>L',len(index)), index ] 927 | 928 | req.append ( pack('>L',len(attrs)) ) 929 | mva_attr = 0 930 | if mva: mva_attr = 1 931 | for attr in attrs: 932 | req.append ( pack('>L',len(attr)) + attr ) 933 | req.append ( pack('>L', mva_attr ) ) 934 | 935 | req.append ( pack('>L',len(values)) ) 936 | for docid, entry in values.items(): 937 | req.append ( pack('>Q',docid) ) 938 | for val in entry: 939 | val_len = val 940 | if mva: val_len = len ( val ) 941 | req.append ( pack('>L',val_len ) ) 942 | if mva: 943 | for vals in val: 944 | req.append ( pack ('>L',vals) ) 945 | 946 | # connect, send query, get response 947 | sock = self._Connect() 948 | if not sock: 949 | return None 950 | 951 | req = ''.join(req) 952 | length = len(req) 953 | req = pack ( '>2HL', SEARCHD_COMMAND_UPDATE, VER_COMMAND_UPDATE, length ) + req 954 | wrote = sock.send ( req ) 955 | 956 | response = self._GetResponse ( sock, VER_COMMAND_UPDATE ) 957 | if not response: 958 | return -1 959 | 960 | # parse response 961 | updated = unpack ( '>L', response[0:4] )[0] 962 | return updated 963 | 964 | 965 | def BuildKeywords ( self, query, index, hits ): 966 | """ 967 | Connect to searchd server, and generate keywords list for a given query. 968 | Returns None on failure, or a list of keywords on success. 969 | """ 970 | assert ( isinstance ( query, str ) ) 971 | assert ( isinstance ( index, str ) ) 972 | assert ( isinstance ( hits, int ) ) 973 | 974 | # build request 975 | req = [ pack ( '>L', len(query) ) + query ] 976 | req.append ( pack ( '>L', len(index) ) + index ) 977 | req.append ( pack ( '>L', hits ) ) 978 | 979 | # connect, send query, get response 980 | sock = self._Connect() 981 | if not sock: 982 | return None 983 | 984 | req = ''.join(req) 985 | length = len(req) 986 | req = pack ( '>2HL', SEARCHD_COMMAND_KEYWORDS, VER_COMMAND_KEYWORDS, length ) + req 987 | wrote = sock.send ( req ) 988 | 989 | response = self._GetResponse ( sock, VER_COMMAND_KEYWORDS ) 990 | if not response: 991 | return None 992 | 993 | # parse response 994 | res = [] 995 | 996 | nwords = unpack ( '>L', response[0:4] )[0] 997 | p = 4 998 | max_ = len(response) 999 | 1000 | while nwords>0 and pL', response[p:p+4] )[0] 1004 | p += 4 1005 | tokenized = response[p:p+length] 1006 | p += length 1007 | 1008 | length = unpack ( '>L', response[p:p+4] )[0] 1009 | p += 4 1010 | normalized = response[p:p+length] 1011 | p += length 1012 | 1013 | entry = { 'tokenized':tokenized, 'normalized':normalized } 1014 | if hits: 1015 | entry['docs'], entry['hits'] = unpack ( '>2L', response[p:p+8] ) 1016 | p += 8 1017 | 1018 | res.append ( entry ) 1019 | 1020 | if nwords>0 or p>max_: 1021 | self._error = 'incomplete reply' 1022 | return None 1023 | 1024 | return res 1025 | 1026 | def Status ( self ): 1027 | """ 1028 | Get the status 1029 | """ 1030 | 1031 | # connect, send query, get response 1032 | sock = self._Connect() 1033 | if not sock: 1034 | return None 1035 | 1036 | req = pack ( '>2HLL', SEARCHD_COMMAND_STATUS, VER_COMMAND_STATUS, 4, 1 ) 1037 | wrote = sock.send ( req ) 1038 | 1039 | response = self._GetResponse ( sock, VER_COMMAND_STATUS ) 1040 | if not response: 1041 | return None 1042 | 1043 | # parse response 1044 | res = [] 1045 | 1046 | p = 8 1047 | max_ = len(response) 1048 | 1049 | while pL', response[p:p+4] )[0] 1051 | k = response[p+4:p+length+4] 1052 | p += 4+length 1053 | length = unpack ( '>L', response[p:p+4] )[0] 1054 | v = response[p+4:p+length+4] 1055 | p += 4+length 1056 | res += [[k, v]] 1057 | 1058 | return res 1059 | 1060 | ### persistent connections 1061 | 1062 | def Open(self): 1063 | if self._socket: 1064 | self._error = 'already connected' 1065 | return None 1066 | 1067 | server = self._Connect() 1068 | if not server: 1069 | return None 1070 | 1071 | # command, command version = 0, body length = 4, body = 1 1072 | request = pack ( '>hhII', SEARCHD_COMMAND_PERSIST, 0, 4, 1 ) 1073 | server.send ( request ) 1074 | 1075 | self._socket = server 1076 | return True 1077 | 1078 | def Close(self): 1079 | if not self._socket: 1080 | self._error = 'not connected' 1081 | return 1082 | self._socket.close() 1083 | self._socket = None 1084 | 1085 | def EscapeString(self, string): 1086 | return re.sub(r"([=\(\)|\-!@~\"&/\\\^\$\=])", r"\\\1", string) 1087 | 1088 | 1089 | def FlushAttributes(self): 1090 | sock = self._Connect() 1091 | if not sock: 1092 | return -1 1093 | 1094 | request = pack ( '>hhI', SEARCHD_COMMAND_FLUSHATTRS, VER_COMMAND_FLUSHATTRS, 0 ) # cmd, ver, bodylen 1095 | sock.send ( request ) 1096 | 1097 | response = self._GetResponse ( sock, VER_COMMAND_FLUSHATTRS ) 1098 | if not response or len(response)!=4: 1099 | self._error = 'unexpected response length' 1100 | return -1 1101 | 1102 | tag = unpack ( '>L', response[0:4] )[0] 1103 | return tag 1104 | 1105 | def AssertInt32 ( value ): 1106 | assert(isinstance(value, (int, long))) 1107 | assert(value>=-2**32-1 and value<=2**32-1) 1108 | 1109 | def AssertUInt32 ( value ): 1110 | assert(isinstance(value, (int, long))) 1111 | assert(value>=0 and value<=2**32-1) 1112 | 1113 | # 1114 | # $Id: sphinxapi.py 3087 2012-01-30 23:07:35Z shodan $ 1115 | # 1116 | -------------------------------------------------------------------------------- /wordforms.txt: -------------------------------------------------------------------------------- 1 | cs > centraal 2 | centrum > cs 3 | Centraal Station > CS 4 | Centraal Station > Amsterdam CS 5 | ns > station 6 | busst. > busstation 7 | Den Bosch > 's-Hertogenbosch 8 | burg. > burgemeester 9 | pr. > prins 10 | str. > straat 11 | 1e > eerste 12 | 2e > tweede 13 | 3e > derde 14 | 4e > vierde 15 | Adm. > admiraal 16 | HS > Hollands Spoor 17 | 18 | --------------------------------------------------------------------------------