├── .gitignore ├── gen_city_stats.sh ├── Dockerfile ├── scripts ├── post_import.sql ├── highways-and-places.style ├── process.sh ├── data_overall.sql └── data_center.sql └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | .*.swp 2 | __pycache__/ 3 | venv/ 4 | data*.csv 5 | -------------------------------------------------------------------------------- /gen_city_stats.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -eu 3 | docker rmi cities || true 4 | docker build . -t cities 5 | docker run --rm -v "$(pwd):/out" -ti cities 6 | docker rmi cities 7 | ls -t data*.csv|head -n 2 8 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM postgis/postgis:13-master 2 | 3 | RUN apt-get update && apt-get install -y --no-install-recommends osm2pgsql wget osmium-tool 4 | 5 | COPY ./scripts /scripts 6 | 7 | CMD ["/scripts/process.sh"] 8 | -------------------------------------------------------------------------------- /scripts/post_import.sql: -------------------------------------------------------------------------------- 1 | alter table planet_osm_line add column is_road boolean not null default false; 2 | update planet_osm_line set is_road = true where highway in ( 3 | 'residential', 'unclassified', 'tertiary', 'secondary', 'primary', 'trunk', 'motorway', 4 | 'secondary_link', 'primary_link', 'tertiary_link', 'trunk_link', 'motorway_link' 5 | ); 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # city-mapping-stats 2 | Scripts to calculate stats for city mapping quality in OSM 3 | 4 | ## Запуск 5 | 6 | Должен быть установлен докер. 7 | 8 | Скрипт может работать час-два, а может и быстрее, как повезёт. 9 | 10 | Просто запускаете `gen_city_stats.sh` и ждёте. 11 | 12 | После работы появятся два csv-файла. 13 | 14 | 15 | ## Результат 16 | 17 | Ежедневно сгенерированные отчёты опубликованы тут: 18 | - https://osm-dump.sh21.cc/ 19 | -------------------------------------------------------------------------------- /scripts/highways-and-places.style: -------------------------------------------------------------------------------- 1 | # OsmType Tag DataType Flags 2 | node,way note text delete # These tags can be long but are useless for rendering 3 | node,way source text delete # This indicates that we shouldn't store them 4 | node,way created_by text delete 5 | 6 | node,way highway text linear 7 | node,way place text polygon 8 | node,way name text linear 9 | node,way lanes text linear 10 | node,way lanes:forward text linear 11 | node,way lanes:backward text linear 12 | node,way maxspeed text linear 13 | node,way maxspeed:forward text linear 14 | node,way maxspeed:backward text linear 15 | node,way maxspeed:type text linear 16 | node,way crossing text linear 17 | node,way traffic_calming text linear 18 | way way_area real # This is calculated during import 19 | 20 | -------------------------------------------------------------------------------- /scripts/process.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -eux 3 | 4 | [ ! -d /out ] && echo 'Please mount /out: e.g. with -v "/path/to/local/data:/out"' && exit 1 5 | 6 | cd /scripts 7 | 8 | export POSTGRES_USER="${PGUSER:-postgres}" 9 | export POSTGRES_DATABASE="${PGDATABASE:-postgres}" 10 | export POSTGRES_PASWORD=12345678 11 | export POSTGRES_HOST_AUTH_METHOD=trust 12 | 13 | /usr/local/bin/docker-entrypoint.sh postgres & 14 | 15 | psql=( psql -U "$POSTGRES_USER" -d "$POSTGRES_DATABASE" -v ON_ERROR_STOP=1 ) 16 | 17 | wget --progress=dot:giga -O data.osm.pbf http://download.geofabrik.de/russia-latest.osm.pbf 18 | osmium tags-filter data.osm.pbf w/highway place -o fdata.osm.pbf 19 | rm data.osm.pbf 20 | 21 | for i in `seq 1 120`; do 22 | echo "Waiting for PostgreSQL start, attempt $i..." 23 | ${psql[@]} -c 'select 1;' && break 24 | sleep 1 25 | done 26 | 27 | osm2pgsql --slim --drop --number-processes 4 -G --style=highways-and-places.style \ 28 | -U ${POSTGRES_USER} -d ${POSTGRES_DATABASE} fdata.osm.pbf 29 | 30 | DATE=$(date -d yesterday +%y%m%d_%H%M) 31 | "${psql[@]}" -f post_import.sql 32 | "${psql[@]}" -f data_center.sql > /out/data_center_$DATE.csv 33 | "${psql[@]}" -f data_overall.sql > /out/data_overall_$DATE.csv 34 | -------------------------------------------------------------------------------- /scripts/data_overall.sql: -------------------------------------------------------------------------------- 1 | copy ( 2 | with data as ( 3 | select city.name as city, 4 | sum(st_length(st_transform(r.way, 4326)::geography)) as len, 5 | sum(st_length(st_transform(r.way, 4326)::geography)) filter ( 6 | where r.lanes is not null or r."lanes:forward" is not null or r."lanes:backward" is not null 7 | ) as len_lanes, 8 | sum(st_length(st_transform(r.way, 4326)::geography)) filter ( 9 | where r.maxspeed is not null or r."maxspeed:forward" is not null or 10 | r."maxspeed:backward" is not null or r."maxspeed:type" is not null 11 | ) as len_speed 12 | from planet_osm_polygon city 13 | left join planet_osm_line r on st_intersects(city.way, r.way) 14 | where city.place = 'city' and r.is_road 15 | group by 1 16 | ), points as ( 17 | select city.name as city, 18 | count(*) filter (where p.highway = 'crossing') as cnt_crossings, 19 | count(*) filter (where p.highway = 'crossing' and p.crossing = 'traffic_signals') as cnt_crossings_signals, 20 | count(*) filter (where p.highway = 'crossing' and p.crossing in ('marked', 'uncontrolled', 'zebra')) as cnt_crossings_marked, 21 | count(*) filter (where p.traffic_calming in ('bump', 'hump', 'table')) as cnt_bumps 22 | from planet_osm_polygon city 23 | left join planet_osm_point p on st_contains(city.way, p.way) 24 | where city.place = 'city' 25 | group by 1 26 | ) 27 | select city, 28 | round(len/1000) as len_km, 29 | round((100*len_lanes/len)::numeric, 2) as perc_lanes, 30 | round((100*len_speed/len)::numeric, 2) as perc_speed, 31 | round((cnt_crossings / len * 1000)::numeric, 4) as crossings_per_km, 32 | round((cnt_crossings_signals / len * 1000)::numeric, 4) as signals_per_km, 33 | round((cnt_crossings_marked / len * 1000)::numeric, 4) as marked_per_km, 34 | round((cnt_bumps / len * 1000)::numeric, 4) as bumps_per_km 35 | from data left join points using (city) 36 | order by city 37 | -- order by len_km desc limit 10 38 | ) to stdout (format csv, header); 39 | -------------------------------------------------------------------------------- /scripts/data_center.sql: -------------------------------------------------------------------------------- 1 | copy ( 2 | with data as ( 3 | select city.name as city, 4 | sum(st_length(st_transform(r.way, 4326)::geography)) as len, 5 | sum(st_length(st_transform(r.way, 4326)::geography)) filter ( 6 | where r.lanes is not null or r."lanes:forward" is not null or r."lanes:backward" is not null 7 | ) as len_lanes, 8 | sum(st_length(st_transform(r.way, 4326)::geography)) filter ( 9 | where r.maxspeed is not null or r."maxspeed:forward" is not null or 10 | r."maxspeed:backward" is not null or r."maxspeed:type" is not null 11 | ) as len_speed 12 | from planet_osm_point city 13 | left join planet_osm_line r on st_dwithin(city.way, r.way, 5000) 14 | where city.place = 'city' and r.is_road 15 | group by 1 16 | ), points as ( 17 | select city.name as city, 18 | count(*) filter (where p.highway = 'crossing') as cnt_crossings, 19 | count(*) filter (where p.highway = 'crossing' and p.crossing = 'traffic_signals') as cnt_crossings_signals, 20 | count(*) filter (where p.highway = 'crossing' and p.crossing in ('marked', 'uncontrolled', 'zebra')) as cnt_crossings_marked, 21 | count(*) filter (where p.traffic_calming in ('bump', 'hump', 'table')) as cnt_bumps 22 | from planet_osm_point city 23 | left join planet_osm_point p on st_dwithin(city.way, p.way, 5000) 24 | where city.place = 'city' 25 | group by 1 26 | ) 27 | select city, 28 | round(len/1000) as len_km, 29 | round((100*len_lanes/len)::numeric, 2) as perc_lanes, 30 | round((100*len_speed/len)::numeric, 2) as perc_speed, 31 | round((cnt_crossings / len * 1000)::numeric, 4) as crossings_per_km, 32 | round((cnt_crossings_signals / len * 1000)::numeric, 4) as signals_per_km, 33 | round((cnt_crossings_marked / len * 1000)::numeric, 4) as marked_per_km, 34 | round((cnt_bumps / len * 1000)::numeric, 4) as bumps_per_km 35 | from data left join points using (city) 36 | order by city 37 | -- order by len_km desc limit 10 38 | ) to stdout (format csv, header); 39 | --------------------------------------------------------------------------------