├── requirements.txt
├── .gitignore
├── db.py
├── setstate.py
├── readplanet.py
├── bigmmap.py
├── changelib.py
└── changechange.py


/requirements.txt:
--------------------------------------------------------------------------------
1 | imposm.parser
2 | lxml
3 | peewee
4 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | 00?/
2 | *.gz
3 | *.pbf
4 | *.pyc
5 | nodes.bin
6 | ways.bin
7 | changechange.db
8 | 


--------------------------------------------------------------------------------
/db.py:
--------------------------------------------------------------------------------
 1 | from peewee import *
 2 | 
 3 | database = SqliteDatabase(None)
 4 | 
 5 | class BaseModel(Model):
 6 |     class Meta:
 7 |         database = database
 8 | 
 9 | class State(BaseModel):
10 |     changeset = IntegerField()
11 |     replication = IntegerField()
12 | 
13 | class Changeset(BaseModel):
14 |     changeset = IntegerField(unique=True)
15 |     timestamp = DateTimeField(index=True)
16 |     xml = TextField()
17 | 
18 | class NodeRef(BaseModel):
19 |     node_id = IntegerField(unique=True)
20 |     refs = CharField(max_length=250)
21 | 
22 | class WayRelRef(BaseModel):
23 |     wr_id = IntegerField(unique=True)
24 |     refs = CharField(max_length=250)
25 | 
26 | class Members(BaseModel):
27 |     wr_id = IntegerField(unique=True)
28 |     members = TextField()
29 | 


--------------------------------------------------------------------------------
/setstate.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | import sys, re, os
 3 | from db import database, State
 4 | 
 5 | def parse_state(s):
 6 |     if s.isdigit():
 7 |         return int(s)
 8 |     m = re.search(r'\.org/replication.*/(\d{3})/(\d{3})/(\d{3})\.', s)
 9 |     if m:
10 |         return int(m.group(1) + m.group(2) + m.group(3))
11 |     return 0
12 | 
13 | if len(sys.argv) < 3:
14 |     print 'Set state values for changechange.'
15 |     print 'Usage: {0} <changeset_state> <replication_state> [path_to_db]'.format(sys.argv[0])
16 |     sys.exit(1)
17 | 
18 | changeset = parse_state(sys.argv[1])
19 | replication = parse_state(sys.argv[2])
20 | if changeset < 1800000 or replication < 1800000:
21 |     print 'Too old replication values:', changeset, replication
22 |     sys.exit(1)
23 | 
24 | path = os.path.dirname(sys.argv[0]) if len(sys.argv) < 4 else sys.argv[3]
25 | database.init(os.path.join(path, 'changechange.db'))
26 | database.connect()
27 | database.create_tables([State], safe=True)
28 | try:
29 |     st = State.get(State.id == 1)
30 | except State.DoesNotExist:
31 |     st = State()
32 | st.changeset = changeset
33 | st.replication = replication
34 | st.save()
35 | 


--------------------------------------------------------------------------------
/readplanet.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | import changelib, sys, os, subprocess
 3 | from db import database, NodeRef, WayRelRef, Members
 4 | from imposm.parser import OSMParser
 5 | 
 6 | NODE_COUNT = 5000*1024*1024
 7 | WAY_COUNT = NODE_COUNT / 10
 8 | 
 9 | if len(sys.argv) < 2:
10 |     print 'Imports planet data into binary files for changechange.'
11 |     print 'Usage: {0} <planet_file.osm.pbf> [<path_to_db>]'.format(sys.argv[0])
12 |     sys.exit(1)
13 | 
14 | path = os.path.dirname(sys.argv[0]) if len(sys.argv) < 3 or not os.path.exists(sys.argv[2]) else sys.argv[2]
15 | database.init(os.path.join(path, 'changechange.db'))
16 | 
17 | if not os.path.exists(os.path.join(path, 'nodes.bin')):
18 |     print 'Creating files'
19 |     BLOCK_SIZE = 16*1024*1024
20 |     for name in (('nodes.bin', NODE_COUNT*12), ('ways.bin', WAY_COUNT*16)):
21 |         res = subprocess.call(['dd', 'if=/dev/zero', 'of='+os.path.join(path, name[0]), 'bs={0}'.format(BLOCK_SIZE), 'count={0}'.format(name[1]/BLOCK_SIZE)])
22 |         if res != 0:
23 |             print 'DD returned code', res
24 |             sys.exit(1)
25 | 
26 | 
27 | class ParserForChange():
28 |     def __init__(self):
29 |         self.cnt = 0
30 | 
31 |     def flush(self):
32 |         self.cnt += 1
33 |         if self.cnt > 1000000:
34 |             changelib.flush()
35 |             self.cnt = 0
36 | 
37 |     def print_state(self, typ, ident):
38 |         sys.stdout.write('\rProcessing {0} {1:.1f}m{2}'.format(typ, ident / 1000000.0, ' ' * 10))
39 |         sys.stdout.flush()
40 |         self.flush()
41 | 
42 |     def got_coords(self, coords_list):
43 |         for coords in coords_list:
44 |             self.print_state('node', coords[0])
45 |             changelib.store_node_coords_fast(coords[0], coords[2], coords[1])
46 | 
47 |     def got_way(self, ways):
48 |         for way in ways:
49 |             self.print_state('way', way[0])
50 |             changelib.update_way_nodes(way[0], way[2])
51 | 
52 |     def got_relation(self, relations):
53 |         for rel in relations:
54 |             self.print_state('relation', rel[0])
55 |             members = [x[1][0] + str(x[0]) for x in rel[2]]
56 |             changelib.update_relation_members(rel[0], members)
57 | 
58 | database.connect()
59 | database.create_tables([NodeRef, WayRelRef, Members], safe=True)
60 | changelib.CACHE = False
61 | changelib.open(path)
62 | p = ParserForChange()
63 | op = OSMParser(concurrency=1, coords_callback=p.got_coords,
64 |                ways_callback=p.got_way, relations_callback=p.got_relation)
65 | op.parse(sys.argv[1])
66 | print
67 | changelib.close()
68 | 


--------------------------------------------------------------------------------
/bigmmap.py:
--------------------------------------------------------------------------------
  1 | import mmap
  2 | import struct
  3 | from collections import deque, Counter
  4 | 
  5 | 
  6 | class BigMMap:
  7 |     """An mmap of int32 numbers contained in a very big file."""
  8 |     ZERO_VALUE = 0x7FFFFFFE
  9 | 
 10 |     def __init__(self, filename, mmap_count=2, page_size=64):
 11 |         self.page_size = page_size * 1024 * 1024
 12 |         self.mmap_count = mmap_count
 13 |         self.history_size = 1000
 14 |         try:
 15 |             self.f = open(filename, 'r+b')
 16 |             self.f.seek(0, 2)
 17 |             self.length = self.f.tell()
 18 |         except IOError:
 19 |             self.f = None
 20 |             self.length = 0
 21 |         self.map = {}
 22 |         self.access_count = Counter()
 23 |         self.accessed_pages = deque()
 24 | 
 25 |     def flush(self, page=None):
 26 |         if page is not None:
 27 |             if page in self.map:
 28 |                 self.map[page].flush()
 29 |         else:
 30 |             for i in self.map:
 31 |                 self.map[i].flush()
 32 | 
 33 |     def close(self, page=None):
 34 |         self.flush(page)
 35 |         if page is not None:
 36 |             if page in self.map:
 37 |                 self.map[page].close()
 38 |                 del self.map[page]
 39 |         else:
 40 |             for m in self.map:
 41 |                 self.map[m].close()
 42 |             self.map.clear()
 43 |             if self.f is not None:
 44 |                 self.f.close()
 45 | 
 46 |     def _get_page(self, offset):
 47 |         """Returns a tuple (mmap, adj. offset)."""
 48 |         if (offset << 2) + 4 >= self.length:
 49 |             raise ValueError('Offset {0} is outside the file length {1}.'.format(offset, self.length >> 2))
 50 |         page = offset / self.page_size
 51 |         if page not in self.map:
 52 |             if len(self.map) >= self.mmap_count:
 53 |                 # Find the least used map
 54 |                 usage = self.history_size
 55 |                 m = None
 56 |                 for i in self.map:
 57 |                     if self.access_count[i] < usage:
 58 |                         m = i
 59 |                         usage = self.access_count[i]
 60 |                 self.close(m)
 61 |             fofs = page * self.page_size << 2
 62 |             flen = min(self.page_size << 2, self.length - fofs)
 63 |             self.map[page] = mmap.mmap(self.f.fileno(), flen, offset=fofs)
 64 |         # Update counts
 65 |         self.access_count[page] += 1
 66 |         self.accessed_pages.append(page)
 67 |         while len(self.accessed_pages) > self.history_size:
 68 |             self.access_count[self.accessed_pages.popleft()] -= 1
 69 |         return (self.map[page], (offset - page * self.page_size) << 2)
 70 | 
 71 |     def __len__(self):
 72 |         return self.length >> 2
 73 | 
 74 |     def __getitem__(self, offset):
 75 |         if self.f is None:
 76 |             return None
 77 |         m = self._get_page(offset)
 78 |         s = m[0][m[1]:m[1] + 4]
 79 |         v = struct.unpack('<l', s)[0]
 80 |         if v == 0:
 81 |             return None
 82 |         elif v == self.ZERO_VALUE:
 83 |             return 0
 84 |         else:
 85 |             return v
 86 | 
 87 |     def __setitem__(self, offset, value):
 88 |         if self.f is None:
 89 |             return
 90 |         if value is None:
 91 |             v = 0
 92 |         elif value == 0:
 93 |             v = self.ZERO_VALUE
 94 |         else:
 95 |             v = value
 96 |         try:
 97 |             s = struct.pack('<l', v)
 98 |         except struct.error as e:
 99 |             print 'Erroneous value:', v
100 |             raise e
101 |         m = self._get_page(offset)
102 |         m[0][m[1]:m[1] + 4] = s
103 | 


--------------------------------------------------------------------------------
/changelib.py:
--------------------------------------------------------------------------------
  1 | from bigmmap import BigMMap
  2 | from db import NodeRef, WayRelRef, Members
  3 | from os.path import join
  4 | 
  5 | CACHE = True
  6 | COORD_MULTIPLIER = 1e7
  7 | node_mmap = None
  8 | bbox_mmap = None
  9 | 
 10 | NODE_CACHE_MAX_SIZE = 100000
 11 | node_cache = {}
 12 | last_nodes = []
 13 | 
 14 | BBOX_CACHE_MAX_SIZE = 10000
 15 | bbox_cache = {}
 16 | last_bboxes = []
 17 | 
 18 | 
 19 | def open(path):
 20 |     global node_mmap, bbox_mmap
 21 |     node_mmap = BigMMap(join(path, 'nodes.bin'))
 22 |     bbox_mmap = BigMMap(join(path, 'ways.bin'))
 23 | 
 24 | 
 25 | def flush():
 26 |     node_mmap.flush()
 27 |     bbox_mmap.flush()
 28 | 
 29 | 
 30 | def close():
 31 |     node_mmap.close()
 32 |     bbox_mmap.close()
 33 | 
 34 | 
 35 | def coord_to_int32(coord):
 36 |     return None if coord is None else int(round(coord * COORD_MULTIPLIER))
 37 | 
 38 | 
 39 | def int32_to_coord(value):
 40 |     return None if value is None else float(value) / COORD_MULTIPLIER
 41 | 
 42 | 
 43 | def split_comma(s):
 44 |     if len(s) == 0:
 45 |         return []
 46 |     return s.split(',')
 47 | 
 48 | 
 49 | def split_comma_i(s):
 50 |     return [int(x) for x in split_comma(s)]
 51 | 
 52 | 
 53 | def fetch_node_tuple(node_id):
 54 |     if CACHE and node_id in node_cache:
 55 |         return node_cache[node_id]
 56 |     base = node_id * 3
 57 |     lat = int32_to_coord(node_mmap[base])
 58 |     lon = int32_to_coord(node_mmap[base + 1])
 59 |     ref = node_mmap[base + 2]
 60 |     t = (lat, lon, ref)
 61 |     if CACHE:
 62 |         node_cache[node_id] = t
 63 |     last_nodes.append(node_id)
 64 |     return t
 65 | 
 66 | 
 67 | def store_node_coords_fast(node_id, lat, lon):
 68 |     base = node_id * 3
 69 |     node_mmap[base] = coord_to_int32(lat)
 70 |     node_mmap[base + 1] = coord_to_int32(lon)
 71 | 
 72 | 
 73 | def store_node_coords(node_id, lat, lon):
 74 |     t = fetch_node_tuple(node_id)
 75 |     if lat == t[0] and lon == t[1]:
 76 |         return
 77 |     if CACHE:
 78 |         n = (lat, lon, t[2])
 79 |         node_cache[node_id] = n
 80 |     base = node_id * 3
 81 |     node_mmap[base] = coord_to_int32(lat)
 82 |     node_mmap[base + 1] = coord_to_int32(lon)
 83 |     for ref in fetch_node_refs(node_id):
 84 |         if ref > 0:
 85 |             update_way_bbox(ref)
 86 | 
 87 | 
 88 | def fetch_way_bbox(way_id):
 89 |     if CACHE and way_id in bbox_cache:
 90 |         return bbox_cache[way_id]
 91 |     base = way_id * 4
 92 |     bbox = [int32_to_coord(bbox_mmap[base + x]) for x in range(4)]
 93 |     if bbox[0] is None or bbox[1] is None or bbox[2] is None:
 94 |         return None
 95 |     if CACHE:
 96 |         bbox_cache[way_id] = bbox
 97 |     last_bboxes.append(way_id)
 98 |     return bbox
 99 | 
100 | 
101 | def store_way_bbox(way_id, bbox):
102 |     if bbox is None:
103 |         return
104 |     if CACHE:
105 |         if way_id not in bbox_cache:
106 |             last_bboxes.append(way_id)
107 |         bbox_cache[way_id] = bbox
108 |     base = way_id * 4
109 |     for n in range(4):
110 |         bbox_mmap[base + n] = coord_to_int32(bbox[n])
111 | 
112 | 
113 | def add_node_ref(node_id, wr_id):
114 |     t = fetch_node_tuple(node_id)
115 |     if t[2] is None:
116 |         if CACHE:
117 |             t = (t[0], t[1], wr_id)
118 |             node_cache[node_id] = t
119 |         node_mmap[node_id * 3 + 2] = wr_id
120 |     elif t[2] != wr_id:
121 |         try:
122 |             nr = NodeRef.get(NodeRef.node_id == node_id)
123 |             refs = set(split_comma_i(nr.refs))
124 |             if wr_id not in refs:
125 |                 refs.add(wr_id)
126 |                 nr.refs = ','.join([str(x) for x in refs])
127 |                 nr.save()
128 |         except NodeRef.DoesNotExist:
129 |             nr = NodeRef()
130 |             nr.node_id = node_id
131 |             nr.refs = str(wr_id)
132 |             nr.save()
133 | 
134 | 
135 | def remove_node_ref(node_id, wr_id):
136 |     t = fetch_node_tuple(node_id)
137 |     refs = set(fetch_node_refs(node_id))
138 |     if wr_id in refs:
139 |         refs.remove(wr_id)
140 |     were_refs = len(refs) > 0
141 |     if wr_id == t[2]:
142 |         newval = None if len(refs) == 0 else refs.pop()
143 |         t = (t[0], t[1], newval)
144 |         if CACHE:
145 |             node_cache[node_id] = t
146 |         node_mmap[node_id * 3 + 2] = newval
147 |         if were_refs:
148 |             nr = NodeRef.get(NodeRef.node_id == node_id)
149 |             if len(refs) > 0:
150 |                 nr.refs = ','.join([str(x) for x in refs])
151 |                 nr.save()
152 |             else:
153 |                 nr.delete_instance()
154 | 
155 | 
156 | def fetch_node_refs(node_id):
157 |     refs = []
158 |     t = fetch_node_tuple(node_id)
159 |     if t[2] is not None:
160 |         refs.append(t[2])
161 |         try:
162 |             nr = NodeRef.get(NodeRef.node_id == node_id)
163 |             refs.extend(split_comma_i(nr.refs))
164 |         except NodeRef.DoesNotExist:
165 |             pass
166 |     return refs
167 | 
168 | 
169 | def add_wr_ref(wr_id, ref_id):
170 |     try:
171 |         wr = WayRelRef.get(WayRelRef.wr_id == wr_id)
172 |     except WayRelRef.DoesNotExist:
173 |         wr = WayRelRef()
174 |         wr.wr_id = wr_id
175 |         wr.refs = ''
176 |     refs = split_comma(wr.refs)
177 |     try:
178 |         refs.index(str(-ref_id))
179 |     except ValueError:
180 |         wr.refs = ','.join(refs + [str(-ref_id)])
181 |         wr.save()
182 | 
183 | 
184 | def remove_wr_ref(wr_id, ref_id):
185 |     try:
186 |         wr = WayRelRef.get(WayRelRef.wr_id == wr_id)
187 |         refs = split_comma(wr.refs)
188 |         try:
189 |             refs.remove(str(-ref_id))
190 |             wr.refs = ','.join(refs)
191 |             wr.save()
192 |         except ValueError:
193 |             pass
194 |     except WayRelRef.DoesNotExist:
195 |         pass
196 | 
197 | 
198 | def fetch_wr_refs(wr_id):
199 |     try:
200 |         wr = WayRelRef.get(WayRelRef.wr_id == wr_id)
201 |         return [-int(x) for x in split_comma(wr.refs)]
202 |     except WayRelRef.DoesNotExist:
203 |         return []
204 | 
205 | 
206 | def fetch_way_nodes(way_id):
207 |     try:
208 |         member = Members.get(Members.wr_id == way_id)
209 |         return split_comma_i(member.members)
210 |     except Members.DoesNotExist:
211 |         return []
212 | 
213 | 
214 | def calc_bbox(nodes):
215 |     bbox = None
216 |     for n in nodes:
217 |         t = fetch_node_tuple(n)
218 |         if t[0] is not None and t[1] is not None:
219 |             if bbox is None:
220 |                 bbox = [t[0], t[1], t[0], t[1]]
221 |             else:
222 |                 bbox[0] = min(bbox[0], t[0])
223 |                 bbox[1] = min(bbox[1], t[1])
224 |                 bbox[2] = max(bbox[0], t[0])
225 |                 bbox[3] = max(bbox[1], t[1])
226 |     return bbox
227 | 
228 | 
229 | def update_way_nodes(way_id, nodes):
230 |     # Update way members in the database
231 |     try:
232 |         way = Members.get(Members.wr_id == way_id)
233 |     except Members.DoesNotExist:
234 |         way = Members()
235 |         way.wr_id = way_id
236 |         way.members = ''
237 |     new_members = ','.join([str(x) for x in nodes])
238 |     if new_members == way.members:
239 |         return
240 |     old_nodes = set(split_comma_i(way.members))
241 |     way.members = new_members
242 |     way.save()
243 |     # Update stored way bbox
244 |     bbox = calc_bbox(nodes)
245 |     store_way_bbox(way_id, bbox)
246 |     # Update references for nodes
247 |     for n in nodes:
248 |         if n not in old_nodes:
249 |             add_node_ref(n, way_id)
250 |         else:
251 |             old_nodes.remove(n)
252 |     for n in old_nodes:
253 |         remove_node_ref(n, way_id)
254 | 
255 | 
256 | def update_way_bbox(way_id):
257 |     nodes = fetch_way_nodes(way_id)
258 |     bbox = calc_bbox(nodes)
259 |     store_way_bbox(way_id, bbox)
260 | 
261 | 
262 | def update_relation_members(rel_id, members):
263 |     try:
264 |         rel = Members.get(Members.wr_id == rel_id)
265 |     except Members.DoesNotExist:
266 |         rel = Members()
267 |         rel.wr_id = rel_id
268 |         rel.members = ''
269 |     new_members = ','.join(members)
270 |     if new_members == rel.members:
271 |         return
272 |     old_members = set(split_comma(rel.members))
273 |     rel.members = new_members
274 |     rel.save()
275 |     # Update references for individual objects
276 |     for m in members:
277 |         typ = m[0]
278 |         ref = int(m[1:])
279 |         if typ == 'n':
280 |             add_node_ref(ref, rel_id)
281 |         else:
282 |             add_wr_ref(ref, rel_id)
283 |         try:
284 |             old_members.remove(m)
285 |         except KeyError:
286 |             pass
287 |     for m in old_members:
288 |         typ = m[0]
289 |         ref = int(m[1:])
290 |         if typ == 'n':
291 |             remove_node_ref(ref, rel_id)
292 |         else:
293 |             remove_wr_ref(ref, rel_id)
294 | 
295 | 
296 | def delete_wr(wr_id):
297 |     if wr_id > 0:
298 |         update_way_nodes(wr_id, [])
299 |     else:
300 |         update_relation_members(wr_id, [])
301 | 
302 | 
303 | def purge_node_cache():
304 |     global last_nodes, last_bboxes
305 |     if len(last_nodes) > NODE_CACHE_MAX_SIZE:
306 |         for i in range(0, len(last_nodes) - NODE_CACHE_MAX_SIZE):
307 |             del node_cache[last_nodes[i]]
308 |         last_nodes = last_nodes[len(last_nodes) - NODE_CACHE_MAX_SIZE:]
309 |     if len(last_bboxes) > BBOX_CACHE_MAX_SIZE:
310 |         for i in range(0, len(last_bboxes) - BBOX_CACHE_MAX_SIZE):
311 |             del bbox_cache[last_bboxes[i]]
312 |         last_bboxes = last_bboxes[len(last_bboxes) - BBOX_CACHE_MAX_SIZE:]
313 | 


--------------------------------------------------------------------------------
/changechange.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | import os, sys, urllib2, gzip, re, datetime
  3 | from StringIO import StringIO
  4 | from lxml import etree
  5 | import db
  6 | import changelib
  7 | 
  8 | path = os.path.dirname(sys.argv[0]) if len(sys.argv) < 2 or not os.path.exists(sys.argv[1]) else sys.argv[1]
  9 | REPLICATION_BASE_URL = 'http://planet.openstreetmap.org/replication'
 10 | API_BASE_URL = 'http://api.openstreetmap.org/api/0.6'
 11 | TARGET_OSC_PATH = os.path.dirname(sys.argv[0])
 12 | 
 13 | 
 14 | def download_last_state():
 15 |     """Downloads last data and changeset replication seq number."""
 16 |     state = urllib2.urlopen(REPLICATION_BASE_URL + '/minute/state.txt').read()
 17 |     m = re.search(r'sequenceNumber=(\d+)', state)
 18 |     seq1 = int(m.group(1))
 19 | 
 20 |     state = urllib2.urlopen(REPLICATION_BASE_URL + '/changesets/state.yaml').read()
 21 |     m = re.search(r'sequence:\s+(\d+)', state)
 22 |     seq2 = int(m.group(1))
 23 |     # Not checking to throw exception in case of an error
 24 |     return [seq1, seq2]
 25 | 
 26 | 
 27 | def read_last_state():
 28 |     try:
 29 |         st = db.State.get(db.State.id == 1)
 30 |         return [st.replication, st.changeset]
 31 |     except:
 32 |         return None
 33 | 
 34 | 
 35 | def write_last_state(state):
 36 |     try:
 37 |         st = db.State.get(db.State.id == 1)
 38 |     except:
 39 |         st = db.State()
 40 |     st.changeset = state[1]
 41 |     st.replication = state[0]
 42 |     st.save()
 43 | 
 44 | 
 45 | def get_replication_url(state, subdir):
 46 |     return '{0}/{1}/{2:03}/{3:03}/{4:03}.{5}.gz'.format(
 47 |             REPLICATION_BASE_URL,
 48 |             subdir,
 49 |             int(state / 1000000),
 50 |             int(state / 1000) % 1000,
 51 |             state % 1000,
 52 |             'osm' if subdir == 'changesets' else 'osc')
 53 | 
 54 | 
 55 | def get_replication_target_path(state):
 56 |     return os.path.join(TARGET_OSC_PATH, '{0:03}'.format(int(state / 1000000)), '{0:03}'.format(int(state / 1000) % 1000), '{0:03}.osc.gz'.format(state % 1000))
 57 | 
 58 | 
 59 | def process_replication_changesets(state):
 60 |     """Downloads replication archive for a given state, and returns a dict of changeset xml strings."""
 61 |     response = urllib2.urlopen(get_replication_url(state, 'changesets'))
 62 |     data = response.read()
 63 |     gz = gzip.GzipFile(fileobj=StringIO(data))
 64 |     for event, element in etree.iterparse(gz):
 65 |         if element.tag == 'changeset':
 66 |             try:
 67 |                 ch = db.Changeset.get(db.Changeset.changeset == int(element.get('id')))
 68 |             except db.Changeset.DoesNotExist:
 69 |                 ch = db.Changeset()
 70 |                 ch.changeset = int(element.get('id'))
 71 |             ch.timestamp = datetime.datetime.now()
 72 |             ch.xml = etree.tostring(element)
 73 |             ch.save()
 74 |             element.clear()
 75 | 
 76 | 
 77 | def fetch_changeset_from_api(changeset):
 78 |     response = urllib2.urlopen('{0}/changeset/{1}'.format(API_BASE_URL, changeset))
 79 |     return response.read()
 80 | 
 81 | 
 82 | def enrich_replication(state):
 83 |     """Downloads replication archive for a given state, and creates an enriched osc.gz."""
 84 |     response = urllib2.urlopen(get_replication_url(state, 'minute'))
 85 |     data = response.read()
 86 |     gz = gzip.GzipFile(fileobj=StringIO(data))
 87 |     filename = get_replication_target_path(state)
 88 |     if not os.path.exists(os.path.dirname(filename)):
 89 |         os.makedirs(os.path.dirname(filename))
 90 |     gzout = gzip.GzipFile(filename, 'wb')
 91 |     gzout.write("""<?xml version="1.0" encoding="utf-8"?>\n<osmChange version="0.6" generator="Changechange">\n""")
 92 |     action = None
 93 |     printed_changesets = set()
 94 |     for event, element in etree.iterparse(gz, events=('start', 'end')):
 95 |         if element.tag in ('create', 'modify', 'delete') and event == 'start':
 96 |             action = element.tag
 97 |         elif element.tag in ('node', 'way', 'relation') and event == 'end':
 98 |             el_id = int(element.get('id'))
 99 |             if element.tag == 'relation':
100 |                 el_id = -el_id
101 |             # Print changeset if needed
102 |             changeset = int(element.get('changeset'))
103 |             if changeset not in printed_changesets:
104 |                 try:
105 |                     chdata = db.Changeset.get(db.Changeset.changeset == changeset).xml
106 |                 except db.Changeset.DoesNotExist:
107 |                     chdata = fetch_changeset_from_api(changeset)
108 |                 gzout.write(chdata)
109 |                 printed_changesets.add(changeset)
110 | 
111 |             # Add and/or record geometry
112 |             if element.tag == 'node':
113 |                 if element.get('lat'):
114 |                     changelib.store_node_coords(el_id, float(element.get('lat')), float(element.get('lon')))
115 |             elif element.tag == 'way':
116 |                 if action == 'delete' and not element.find('nd'):
117 |                     # Add nodes to deleted ways, so their geometry is not empty
118 |                     for n in changelib.fetch_way_nodes(el_id):
119 |                         ndel = etree.Element('nd')
120 |                         ndel.set('ref', str(n))
121 |                         element.append(ndel)
122 |                 nodes = []
123 |                 for nd in element.findall('nd'):
124 |                     ref = int(nd.get('ref'))
125 |                     nodes.append(ref)
126 |                     node_data = changelib.fetch_node_tuple(ref)
127 |                     if node_data is not None and node_data[0] is not None and node_data[1] is not None:
128 |                         # We leave the possibility of an absent node
129 |                         nd.set('lat', str(node_data[0]))
130 |                         nd.set('lon', str(node_data[1]))
131 |                 if action != 'delete':
132 |                     changelib.update_way_nodes(el_id, nodes)
133 |                 else:
134 |                     changelib.delete_wr(el_id)
135 |             elif element.tag == 'relation':
136 |                 # We are not adding members to deleted relations, since we don't know their roles
137 |                 members = []
138 |                 for member in element.findall('member'):
139 |                     members.append(member.get('type')[0] + member.get('ref'))
140 |                     ref = int(member.get('ref'))
141 |                     if member.get('type') == 'node':
142 |                         node_data = changelib.fetch_node_tuple(ref)
143 |                         if node_data is not None and node_data[0] is not None and node_data[1] is not None:
144 |                             member.set('lat', str(node_data[0]))
145 |                             member.set('lon', str(node_data[1]))
146 |                     elif member.get('type') == 'way':
147 |                         bbox = changelib.fetch_way_bbox(ref)
148 |                         if bbox is not None:
149 |                             member.set('minlat', str(bbox[0]))
150 |                             member.set('minlon', str(bbox[1]))
151 |                             member.set('maxlat', str(bbox[2]))
152 |                             member.set('maxlon', str(bbox[3]))
153 |                 if action != 'delete':
154 |                     changelib.update_relation_members(el_id, members)
155 |                 else:
156 |                     changelib.delete_wr(el_id)
157 | 
158 |             # Add referencing objects
159 |             if element.tag == 'node':
160 |                 refs = changelib.fetch_node_refs(el_id)
161 |             else:
162 |                 refs = changelib.fetch_wr_refs(el_id)
163 |             for ref in refs:
164 |                 refel = etree.Element('ref')
165 |                 refel.set('type', 'way' if ref > 0 else 'relation')
166 |                 refel.set('ref', str(ref))
167 |                 element.append(refel)
168 |             # Print and forget
169 |             p = etree.Element(action)
170 |             p.append(element)
171 |             gzout.write(etree.tostring(p, encoding='utf-8'))
172 |             element.clear()
173 |             p.clear()
174 |     changelib.purge_node_cache()
175 |     changelib.flush()
176 | 
177 | if __name__ == '__main__':
178 |     try:
179 |         cur_state = download_last_state()
180 |     except Exception as e:
181 |         print 'Failed to download last state:', e
182 |         sys.exit(1)
183 | 
184 |     db.database.init(os.path.join(path, 'changechange.db'))
185 |     db.database.connect()
186 |     db.database.create_tables([db.Changeset, db.NodeRef, db.WayRelRef, db.Members, db.State], safe=True)
187 | 
188 |     state = read_last_state()
189 |     if state is None:
190 |         state = [x-1 for x in cur_state]
191 | 
192 |     # Delete old changesets
193 |     tooold = datetime.datetime.now() - datetime.timedelta(days=2)
194 |     query = db.Changeset.delete().where(db.Changeset.timestamp < tooold)
195 |     query.execute()
196 | 
197 |     # Process changeset replication
198 |     sys.stdout.write('Downloading changesets')
199 |     with db.database.atomic():
200 |         while state[1] < cur_state[1]:
201 |             sys.stdout.write('.')
202 |             sys.stdout.flush()
203 |             state[1] += 1
204 |             process_replication_changesets(state[1])
205 |             write_last_state(state)
206 |     print
207 | 
208 |     # Process data replication
209 |     changelib.open(path)
210 |     while state[0] < cur_state[0] - 1:
211 |         with db.database.atomic():
212 |             state[0] += 1
213 |             print state[0]
214 |             enrich_replication(state[0])
215 |             write_last_state(state)
216 |     changelib.close()
217 | 


--------------------------------------------------------------------------------