├── .github └── workflows │ └── sanity_checks.yml ├── .gitignore ├── COPYING ├── Cargo.toml ├── README.md ├── build.rs ├── examples ├── count.rs ├── count_with_deps.rs ├── relation.rs └── tutorial.rs ├── protos ├── fileformat.proto └── osmformat.proto └── src ├── blobs.rs ├── blocks.rs ├── error.rs ├── groups.rs ├── lib.rs ├── objects.rs └── reader.rs /.github/workflows/sanity_checks.yml: -------------------------------------------------------------------------------- 1 | name: Sanity Checks 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | build: 7 | runs-on: ubuntu-latest 8 | 9 | steps: 10 | - name: Checkout code 11 | uses: actions/checkout@v4 12 | 13 | - name: Set up Rust 14 | uses: actions-rs/toolchain@v1 15 | with: 16 | toolchain: stable 17 | components: rustfmt, clippy 18 | 19 | - name: Check formatting 20 | run: cargo fmt --all -- --check 21 | 22 | - name: Run Clippy 23 | run: cargo clippy -- -Dwarnings 24 | 25 | - name: Run tests 26 | run: cargo test 27 | 28 | - name: Run tests with optional features 29 | run: cargo test --all-features 30 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /Cargo.lock 3 | *~ 4 | /src/fileformat.rs 5 | /src/osmformat.rs 6 | -------------------------------------------------------------------------------- /COPYING: -------------------------------------------------------------------------------- 1 | DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE 2 | Version 2, December 2004 3 | 4 | Copyright (C) 2004 Sam Hocevar 5 | 6 | Everyone is permitted to copy and distribute verbatim or modified 7 | copies of this license document, and changing it is allowed as long 8 | as the name is changed. 9 | 10 | DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE 11 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 12 | 13 | 0. You just DO WHAT THE FUCK YOU WANT TO. 14 | 15 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "osmpbfreader" 3 | version = "0.19.1" 4 | authors = ["Guillaume Pinot "] 5 | description = "Read OpenStreetMap PBF files in rust." 6 | documentation = "https://docs.rs/osmpbfreader" 7 | repository = "https://github.com/TeXitoi/osmpbfreader-rs" 8 | keywords = ["openstreetmap", "osm", "pbf"] 9 | categories = ["encoding", "science"] 10 | license = "WTFPL" 11 | readme = "README.md" 12 | edition = "2021" 13 | 14 | [features] 15 | # Enables serde serialization/deserialization for all OpenStreetMap data structures 16 | serde = ["dep:serde", "flat_map/serde1", "smartstring/serde"] 17 | 18 | [dependencies] 19 | byteorder = "1.3" 20 | flat_map = "0.0.10" 21 | flate2 = "1.0" 22 | par-map = "0.1" 23 | protobuf = "3" 24 | pub-iterator-type = "0.1" 25 | self_cell = "1.0.2" 26 | serde = { version = "1.0", features = ["derive"], optional = true } 27 | smartstring = "1.0.1" 28 | 29 | [dev-dependencies] 30 | log = "0.4" 31 | env_logger = "0.9.0" 32 | 33 | [build-dependencies] 34 | protobuf-codegen = "3" 35 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # osmpbfreader-rs [![](https://img.shields.io/crates/v/osmpbfreader.svg)](https://crates.io/crates/osmpbfreader) [![](https://docs.rs/osmpbfreader/badge.svg)](https://docs.rs/osmpbfreader) 2 | 3 | Read [OpenStreetMap PBF 4 | files](http://wiki.openstreetmap.org/wiki/PBF_Format) with 5 | [rust](http://www.rust-lang.org) in an easy and effective way. 6 | The main inspiration of this library is 7 | [libosmpbfreader](https://github.com/CanalTP/libosmpbfreader). 8 | 9 | ## Documentation 10 | 11 | Find it on [Docs.rs](https://docs.rs/osmpbfreader) 12 | 13 | ## Using this lib 14 | 15 | This crate works with Cargo and is on 16 | [crates.io](https://crates.io/crates/osmpbfreader). The package is regularly 17 | updated. 18 | 19 | For complete example, you can see the [examples](examples/). 20 | 21 | You can find OSM PBF files at [Geofabrik's free download server](http://download.geofabrik.de/). 22 | 23 | ## Performances 24 | 25 | Using the different examples compiled in release mode: 26 | ``` 27 | $ grep CPU /proc/cpuinfo | uniq -c 28 | 8 model name : Intel(R) Core(TM) i7-4702HQ CPU @ 2.20GHz 29 | $ rustc --version 30 | rustc 1.14.0 (e8a012324 2016-12-16) 31 | $ ls -sh france-latest.osm.pbf 32 | 3,3G france-latest.osm.pbf 33 | $ time ./target/release/examples/tutorial france-latest.osm.pbf 34 | 416483839 objects in "france-latest.osm.pbf" 35 | 36 | real 4m24.784s 37 | user 4m18.476s 38 | sys 0m6.164s 39 | $ time ./target/release/examples/count france-latest.osm.pbf admin_level 8 40 | counting objects with tags["admin_level"] = "8"... 41 | 53 nodes, mean coord: 46.25862766415095, 2.9082348867924517. 42 | 108190 ways, mean |nodes|: 72.09304926518162 43 | 35984 relations, mean |references|: 8.705369052912406 44 | 45 | real 1m10.117s 46 | user 8m16.164s 47 | sys 0m23.120s 48 | $ time ./target/release/examples/count_with_deps france-latest.osm.pbf admin_level 8 49 | counting objects with tags["admin_level"] = "8" and their depedencies... 50 | 9497221 nodes, mean coord: 46.69071931974348, 2.2632424769587915. 51 | 136950 ways, mean |nodes|: 70.35282949981745 52 | 36408 relations, mean |references|: 8.771121731487586 53 | 54 | real 5m9.814s 55 | user 33m52.820s 56 | sys 0m28.624s 57 | ``` 58 | 59 | ## License 60 | 61 | This work is free. You can redistribute it and/or modify it under the 62 | terms of the Do What The Fuck You Want To Public License, Version 2, 63 | as published by Sam Hocevar. See the COPYING file for more details. 64 | 65 | Note that `protos/fileformat.proto` and `protos/osmformat.proto` come from 66 | [OSM-binary](https://github.com/scrosby/OSM-binary) under the MIT. 67 | -------------------------------------------------------------------------------- /build.rs: -------------------------------------------------------------------------------- 1 | extern crate protobuf_codegen; 2 | 3 | use std::io::Write; 4 | 5 | static MOD_RS: &[u8] = b" 6 | /// Generated from protobuf. 7 | pub mod fileformat; 8 | 9 | /// Generated from protobuf. 10 | pub mod osmformat; 11 | "; 12 | 13 | fn main() -> Result<(), Box> { 14 | let out_dir = std::env::var("OUT_DIR")?; 15 | 16 | protobuf_codegen::Codegen::new() 17 | .pure() 18 | .out_dir(&out_dir) 19 | .inputs(["protos/fileformat.proto", "protos/osmformat.proto"]) 20 | .include("protos") 21 | .run() 22 | .expect("Codegen failed."); 23 | 24 | std::fs::File::create(out_dir + "/mod.rs")?.write_all(MOD_RS)?; 25 | 26 | Ok(()) 27 | } 28 | -------------------------------------------------------------------------------- /examples/count.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014-2015 Guillaume Pinot 2 | // 3 | // This work is free. You can redistribute it and/or modify it under 4 | // the terms of the Do What The Fuck You Want To Public License, 5 | // Version 2, as published by Sam Hocevar. See the COPYING file for 6 | // more details. 7 | 8 | #[macro_use] 9 | extern crate log; 10 | extern crate env_logger; 11 | extern crate osmpbfreader; 12 | 13 | fn count bool>(filter: F, filename: &std::ffi::OsStr) { 14 | let r = std::fs::File::open(std::path::Path::new(filename)).unwrap(); 15 | let mut pbf = osmpbfreader::OsmPbfReader::new(r); 16 | let mut nb_nodes = 0; 17 | let mut sum_lon = 0.; 18 | let mut sum_lat = 0.; 19 | let mut nb_ways = 0; 20 | let mut nb_way_nodes = 0; 21 | let mut nb_rels = 0; 22 | let mut nb_rel_refs = 0; 23 | for obj in pbf.par_iter().map(Result::unwrap) { 24 | if !filter(obj.tags()) { 25 | continue; 26 | } 27 | info!("{:?}", obj); 28 | match obj { 29 | osmpbfreader::OsmObj::Node(node) => { 30 | nb_nodes += 1; 31 | sum_lon += node.lon(); 32 | sum_lat += node.lat(); 33 | } 34 | osmpbfreader::OsmObj::Way(way) => { 35 | nb_ways += 1; 36 | nb_way_nodes += way.nodes.len(); 37 | } 38 | osmpbfreader::OsmObj::Relation(rel) => { 39 | nb_rels += 1; 40 | nb_rel_refs += rel.refs.len(); 41 | } 42 | } 43 | } 44 | println!( 45 | "{} nodes, mean coord: {}, {}.", 46 | nb_nodes, 47 | sum_lat / nb_nodes as f64, 48 | sum_lon / nb_nodes as f64 49 | ); 50 | println!( 51 | "{} ways, mean |nodes|: {}", 52 | nb_ways, 53 | nb_way_nodes as f64 / nb_ways as f64 54 | ); 55 | println!( 56 | "{} relations, mean |references|: {}", 57 | nb_rels, 58 | nb_rel_refs as f64 / nb_rels as f64 59 | ); 60 | } 61 | 62 | fn main() { 63 | env_logger::init(); 64 | let args: Vec<_> = std::env::args_os().collect(); 65 | match args.len() { 66 | 2 => { 67 | println!("counting objects..."); 68 | count(|_| true, &args[1]); 69 | } 70 | 3 => { 71 | let key = args[2].to_str().unwrap(); 72 | println!("counting objects with \"{}\" in tags...", key); 73 | count(|tags| tags.contains_key(key), &args[1]); 74 | } 75 | 4 => { 76 | let key = args[2].to_str().unwrap(); 77 | let val = args[3].to_str().unwrap(); 78 | println!("counting objects with tags[\"{}\"] = \"{}\"...", key, val); 79 | count(|tags| tags.contains(key, val), &args[1]); 80 | } 81 | _ => println!("usage: count filename [key [value]]",), 82 | }; 83 | } 84 | -------------------------------------------------------------------------------- /examples/count_with_deps.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014-2015 Guillaume Pinot 2 | // 3 | // This work is free. You can redistribute it and/or modify it under 4 | // the terms of the Do What The Fuck You Want To Public License, 5 | // Version 2, as published by Sam Hocevar. See the COPYING file for 6 | // more details. 7 | 8 | #[macro_use] 9 | extern crate log; 10 | extern crate env_logger; 11 | extern crate osmpbfreader; 12 | 13 | fn count bool>(filter: F, filename: &std::ffi::OsStr) { 14 | let r = std::fs::File::open(std::path::Path::new(filename)).unwrap(); 15 | let mut pbf = osmpbfreader::OsmPbfReader::new(r); 16 | let objs = pbf.get_objs_and_deps(|obj| filter(obj.tags())).unwrap(); 17 | let mut nb_nodes = 0; 18 | let mut sum_lon = 0.; 19 | let mut sum_lat = 0.; 20 | let mut nb_ways = 0; 21 | let mut nb_way_nodes = 0; 22 | let mut nb_rels = 0; 23 | let mut nb_rel_refs = 0; 24 | for obj in objs.values() { 25 | info!("{:?}", obj); 26 | match *obj { 27 | osmpbfreader::OsmObj::Node(ref node) => { 28 | nb_nodes += 1; 29 | sum_lon += node.lon(); 30 | sum_lat += node.lat(); 31 | } 32 | osmpbfreader::OsmObj::Way(ref way) => { 33 | nb_ways += 1; 34 | nb_way_nodes += way.nodes.len(); 35 | } 36 | osmpbfreader::OsmObj::Relation(ref rel) => { 37 | nb_rels += 1; 38 | nb_rel_refs += rel.refs.len(); 39 | } 40 | } 41 | } 42 | println!( 43 | "{} nodes, mean coord: {}, {}.", 44 | nb_nodes, 45 | sum_lat / nb_nodes as f64, 46 | sum_lon / nb_nodes as f64 47 | ); 48 | println!( 49 | "{} ways, mean |nodes|: {}", 50 | nb_ways, 51 | nb_way_nodes as f64 / nb_ways as f64 52 | ); 53 | println!( 54 | "{} relations, mean |references|: {}", 55 | nb_rels, 56 | nb_rel_refs as f64 / nb_rels as f64 57 | ); 58 | } 59 | 60 | fn main() { 61 | env_logger::init(); 62 | let args: Vec<_> = std::env::args_os().collect(); 63 | match args.len() { 64 | 3 => { 65 | let key = args[2].to_str().unwrap(); 66 | println!( 67 | "counting objects with \"{}\" in tags and their depedencies...", 68 | key 69 | ); 70 | count(|tags| tags.contains_key(key), &args[1]); 71 | } 72 | 4 => { 73 | let key = args[2].to_str().unwrap(); 74 | let val = args[3].to_str().unwrap(); 75 | println!( 76 | "counting objects with tags[\"{}\"] = \"{}\" and their depedencies...", 77 | key, val 78 | ); 79 | count(|tags| tags.contains(key, val), &args[1]); 80 | } 81 | _ => println!("usage: count filename key [value]",), 82 | }; 83 | } 84 | -------------------------------------------------------------------------------- /examples/relation.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014-2015 Guillaume Pinot 2 | // 3 | // This work is free. You can redistribute it and/or modify it under 4 | // the terms of the Do What The Fuck You Want To Public License, 5 | // Version 2, as published by Sam Hocevar. See the COPYING file for 6 | // more details. 7 | // 8 | extern crate osmpbfreader; 9 | 10 | fn wanted(obj: &osmpbfreader::OsmObj) -> bool { 11 | obj.id() == osmpbfreader::RelationId(7444).into() //id of relation for Paris 12 | } 13 | 14 | fn main() { 15 | let filename = std::env::args_os().nth(1).unwrap(); 16 | let path = std::path::Path::new(&filename); 17 | let r = std::fs::File::open(path).unwrap(); 18 | let mut pbf = osmpbfreader::OsmPbfReader::new(r); 19 | let objects = pbf.get_objs_and_deps(wanted).unwrap(); 20 | println!( 21 | "The relation Paris is composed of {:?} items", 22 | objects.len() 23 | ); 24 | for (id, _) in objects { 25 | println!("{:?}", id); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /examples/tutorial.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014-2015 Guillaume Pinot 2 | // 3 | // This work is free. You can redistribute it and/or modify it under 4 | // the terms of the Do What The Fuck You Want To Public License, 5 | // Version 2, as published by Sam Hocevar. See the COPYING file for 6 | // more details. 7 | 8 | extern crate osmpbfreader; 9 | 10 | fn main() { 11 | let filename = std::env::args_os().nth(1).unwrap(); 12 | let path = std::path::Path::new(&filename); 13 | let r = std::fs::File::open(path).unwrap(); 14 | let mut pbf = osmpbfreader::OsmPbfReader::new(r); 15 | let mut nb = 0; 16 | for _obj in pbf.iter().map(Result::unwrap) { 17 | nb += 1; 18 | } 19 | println!("{} objects in {:?}", nb, filename); 20 | } 21 | -------------------------------------------------------------------------------- /protos/fileformat.proto: -------------------------------------------------------------------------------- 1 | /** Copyright (c) 2010 Scott A. Crosby. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | 9 | */ 10 | 11 | option optimize_for = LITE_RUNTIME; 12 | option java_package = "crosby.binary"; 13 | package OSMPBF; 14 | 15 | //protoc --java_out=../.. fileformat.proto 16 | 17 | 18 | // 19 | // STORAGE LAYER: Storing primitives. 20 | // 21 | 22 | message Blob { 23 | optional bytes raw = 1; // No compression 24 | optional int32 raw_size = 2; // When compressed, the uncompressed size 25 | 26 | // Possible compressed versions of the data. 27 | optional bytes zlib_data = 3; 28 | 29 | // PROPOSED feature for LZMA compressed data. SUPPORT IS NOT REQUIRED. 30 | optional bytes lzma_data = 4; 31 | 32 | // Formerly used for bzip2 compressed data. Depreciated in 2010. 33 | optional bytes OBSOLETE_bzip2_data = 5 [deprecated=true]; // Don't reuse this tag number. 34 | } 35 | 36 | /* A file contains an sequence of fileblock headers, each prefixed by 37 | their length in network byte order, followed by a data block 38 | containing the actual data. types staring with a "_" are reserved. 39 | */ 40 | 41 | message BlobHeader { 42 | required string type = 1; 43 | optional bytes indexdata = 2; 44 | required int32 datasize = 3; 45 | } 46 | 47 | 48 | -------------------------------------------------------------------------------- /protos/osmformat.proto: -------------------------------------------------------------------------------- 1 | /** Copyright (c) 2010 Scott A. Crosby. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | 9 | */ 10 | 11 | option optimize_for = LITE_RUNTIME; 12 | option java_package = "crosby.binary"; 13 | package OSMPBF; 14 | 15 | /* OSM Binary file format 16 | 17 | This is the master schema file of the OSM binary file format. This 18 | file is designed to support limited random-access and future 19 | extendability. 20 | 21 | A binary OSM file consists of a sequence of FileBlocks (please see 22 | fileformat.proto). The first fileblock contains a serialized instance 23 | of HeaderBlock, followed by a sequence of PrimitiveBlock blocks that 24 | contain the primitives. 25 | 26 | Each primitiveblock is designed to be independently parsable. It 27 | contains a string table storing all strings in that block (keys and 28 | values in tags, roles in relations, usernames, etc.) as well as 29 | metadata containing the precision of coordinates or timestamps in that 30 | block. 31 | 32 | A primitiveblock contains a sequence of primitive groups, each 33 | containing primitives of the same type (nodes, densenodes, ways, 34 | relations). Coordinates are stored in signed 64-bit integers. Lat&lon 35 | are measured in units nanodegrees. The default of 36 | granularity of 100 nanodegrees corresponds to about 1cm on the ground, 37 | and a full lat or lon fits into 32 bits. 38 | 39 | Converting an integer to a lattitude or longitude uses the formula: 40 | $OUT = IN * granularity / 10**9$. Many encoding schemes use delta 41 | coding when representing nodes and relations. 42 | 43 | */ 44 | 45 | ////////////////////////////////////////////////////////////////////////// 46 | ////////////////////////////////////////////////////////////////////////// 47 | 48 | /* Contains the file header. */ 49 | 50 | message HeaderBlock { 51 | optional HeaderBBox bbox = 1; 52 | /* Additional tags to aid in parsing this dataset */ 53 | repeated string required_features = 4; 54 | repeated string optional_features = 5; 55 | 56 | optional string writingprogram = 16; 57 | optional string source = 17; // From the bbox field. 58 | 59 | /* Tags that allow continuing an Osmosis replication */ 60 | 61 | // replication timestamp, expressed in seconds since the epoch, 62 | // otherwise the same value as in the "timestamp=..." field 63 | // in the state.txt file used by Osmosis 64 | optional int64 osmosis_replication_timestamp = 32; 65 | 66 | // replication sequence number (sequenceNumber in state.txt) 67 | optional int64 osmosis_replication_sequence_number = 33; 68 | 69 | // replication base URL (from Osmosis' configuration.txt file) 70 | optional string osmosis_replication_base_url = 34; 71 | } 72 | 73 | 74 | /** The bounding box field in the OSM header. BBOX, as used in the OSM 75 | header. Units are always in nanodegrees -- they do not obey 76 | granularity rules. */ 77 | 78 | message HeaderBBox { 79 | required sint64 left = 1; 80 | required sint64 right = 2; 81 | required sint64 top = 3; 82 | required sint64 bottom = 4; 83 | } 84 | 85 | 86 | /////////////////////////////////////////////////////////////////////// 87 | /////////////////////////////////////////////////////////////////////// 88 | 89 | 90 | message PrimitiveBlock { 91 | required StringTable stringtable = 1; 92 | repeated PrimitiveGroup primitivegroup = 2; 93 | 94 | // Granularity, units of nanodegrees, used to store coordinates in this block 95 | optional int32 granularity = 17 [default=100]; 96 | // Offset value between the output coordinates coordinates and the granularity grid in unites of nanodegrees. 97 | optional int64 lat_offset = 19 [default=0]; 98 | optional int64 lon_offset = 20 [default=0]; 99 | 100 | // Granularity of dates, normally represented in units of milliseconds since the 1970 epoch. 101 | optional int32 date_granularity = 18 [default=1000]; 102 | 103 | 104 | // Proposed extension: 105 | //optional BBox bbox = XX; 106 | } 107 | 108 | // Group of OSMPrimitives. All primitives in a group must be the same type. 109 | message PrimitiveGroup { 110 | repeated Node nodes = 1; 111 | optional DenseNodes dense = 2; 112 | repeated Way ways = 3; 113 | repeated Relation relations = 4; 114 | repeated ChangeSet changesets = 5; 115 | } 116 | 117 | 118 | /** String table, contains the common strings in each block. 119 | 120 | Note that we reserve index '0' as a delimiter, so the entry at that 121 | index in the table is ALWAYS blank and unused. 122 | 123 | */ 124 | message StringTable { 125 | repeated bytes s = 1; 126 | } 127 | 128 | /* Optional metadata that may be included into each primitive. */ 129 | message Info { 130 | optional int32 version = 1 [default = -1]; 131 | optional int64 timestamp = 2; 132 | optional int64 changeset = 3; 133 | optional int32 uid = 4; 134 | optional uint32 user_sid = 5; // String IDs 135 | 136 | // The visible flag is used to store history information. It indicates that 137 | // the current object version has been created by a delete operation on the 138 | // OSM API. 139 | // When a writer sets this flag, it MUST add a required_features tag with 140 | // value "HistoricalInformation" to the HeaderBlock. 141 | // If this flag is not available for some object it MUST be assumed to be 142 | // true if the file has the required_features tag "HistoricalInformation" 143 | // set. 144 | optional bool visible = 6; 145 | } 146 | 147 | /** Optional metadata that may be included into each primitive. Special dense format used in DenseNodes. */ 148 | message DenseInfo { 149 | repeated int32 version = 1 [packed = true]; 150 | repeated sint64 timestamp = 2 [packed = true]; // DELTA coded 151 | repeated sint64 changeset = 3 [packed = true]; // DELTA coded 152 | repeated sint32 uid = 4 [packed = true]; // DELTA coded 153 | repeated sint32 user_sid = 5 [packed = true]; // String IDs for usernames. DELTA coded 154 | 155 | // The visible flag is used to store history information. It indicates that 156 | // the current object version has been created by a delete operation on the 157 | // OSM API. 158 | // When a writer sets this flag, it MUST add a required_features tag with 159 | // value "HistoricalInformation" to the HeaderBlock. 160 | // If this flag is not available for some object it MUST be assumed to be 161 | // true if the file has the required_features tag "HistoricalInformation" 162 | // set. 163 | repeated bool visible = 6 [packed = true]; 164 | } 165 | 166 | 167 | // THIS IS STUB DESIGN FOR CHANGESETS. NOT USED RIGHT NOW. 168 | // TODO: REMOVE THIS? 169 | message ChangeSet { 170 | required int64 id = 1; 171 | // 172 | // // Parallel arrays. 173 | // repeated uint32 keys = 2 [packed = true]; // String IDs. 174 | // repeated uint32 vals = 3 [packed = true]; // String IDs. 175 | // 176 | // optional Info info = 4; 177 | 178 | // optional int64 created_at = 8; 179 | // optional int64 closetime_delta = 9; 180 | // optional bool open = 10; 181 | // optional HeaderBBox bbox = 11; 182 | } 183 | 184 | 185 | message Node { 186 | required sint64 id = 1; 187 | // Parallel arrays. 188 | repeated uint32 keys = 2 [packed = true]; // String IDs. 189 | repeated uint32 vals = 3 [packed = true]; // String IDs. 190 | 191 | optional Info info = 4; // May be omitted in omitmeta 192 | 193 | required sint64 lat = 8; 194 | required sint64 lon = 9; 195 | } 196 | 197 | /* Used to densly represent a sequence of nodes that do not have any tags. 198 | 199 | We represent these nodes columnwise as five columns: ID's, lats, and 200 | lons, all delta coded. When metadata is not omitted, 201 | 202 | We encode keys & vals for all nodes as a single array of integers 203 | containing key-stringid and val-stringid, using a stringid of 0 as a 204 | delimiter between nodes. 205 | 206 | ( ( )* '0' )* 207 | */ 208 | 209 | message DenseNodes { 210 | repeated sint64 id = 1 [packed = true]; // DELTA coded 211 | 212 | //repeated Info info = 4; 213 | optional DenseInfo denseinfo = 5; 214 | 215 | repeated sint64 lat = 8 [packed = true]; // DELTA coded 216 | repeated sint64 lon = 9 [packed = true]; // DELTA coded 217 | 218 | // Special packing of keys and vals into one array. May be empty if all nodes in this block are tagless. 219 | repeated int32 keys_vals = 10 [packed = true]; 220 | } 221 | 222 | 223 | message Way { 224 | required int64 id = 1; 225 | // Parallel arrays. 226 | repeated uint32 keys = 2 [packed = true]; 227 | repeated uint32 vals = 3 [packed = true]; 228 | 229 | optional Info info = 4; 230 | 231 | repeated sint64 refs = 8 [packed = true]; // DELTA coded 232 | } 233 | 234 | message Relation { 235 | enum MemberType { 236 | NODE = 0; 237 | WAY = 1; 238 | RELATION = 2; 239 | } 240 | required int64 id = 1; 241 | 242 | // Parallel arrays. 243 | repeated uint32 keys = 2 [packed = true]; 244 | repeated uint32 vals = 3 [packed = true]; 245 | 246 | optional Info info = 4; 247 | 248 | // Parallel arrays 249 | repeated int32 roles_sid = 8 [packed = true]; // This should have been defined as uint32 for consistency, but it is now too late to change it 250 | repeated sint64 memids = 9 [packed = true]; // DELTA encoded 251 | repeated MemberType types = 10 [packed = true]; 252 | } 253 | 254 | -------------------------------------------------------------------------------- /src/blobs.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014-2017 Guillaume Pinot 2 | // 3 | // This work is free. You can redistribute it and/or modify it under 4 | // the terms of the Do What The Fuck You Want To Public License, 5 | // Version 2, as published by Sam Hocevar. See the COPYING file for 6 | // more details. 7 | 8 | //! Iterator and utilities for `fileformat::Blob`. 9 | 10 | use std::iter; 11 | 12 | use crate::blocks::{ 13 | self, Nodes as BlockNodes, OsmObjs as OsmBlockObjs, Relations as BlockRelations, 14 | Ways as BlockWays, 15 | }; 16 | use crate::fileformat::Blob; 17 | use crate::objects::OsmObj; 18 | use crate::osmformat::PrimitiveBlock; 19 | 20 | macro_rules! wrap { 21 | ($name: ident, $wrap_type: ident => $inner_type: path) => { 22 | self_cell::self_cell!( 23 | #[allow(missing_docs)] 24 | pub struct $name { 25 | owner: PrimitiveBlock, 26 | 27 | #[covariant] 28 | dependent: $wrap_type, 29 | } 30 | ); 31 | 32 | impl Iterator for $name { 33 | type Item = $inner_type; 34 | 35 | fn next(&mut self) -> Option { 36 | self.with_dependent_mut(|_, objs| objs.next()) 37 | } 38 | } 39 | }; 40 | } 41 | 42 | wrap!(OsmBlobObjs, OsmBlockObjs => OsmObj); 43 | wrap!(OsmBlobRelations, BlockRelations => super::Relation); 44 | wrap!(OsmBlobWays, BlockWays => super::Way); 45 | wrap!(OsmBlobNodes, BlockNodes => super::Node); 46 | 47 | /// An iterator on `Result`. 48 | pub struct OsmObjs(OsmObjsImpl); 49 | 50 | #[allow(clippy::type_complexity)] 51 | enum OsmObjsImpl { 52 | OkIter(iter::Map::Item) -> crate::Result<::Item>>), 53 | ErrIter(iter::Once::Item>>), 54 | } 55 | 56 | impl Iterator for OsmObjs { 57 | type Item = crate::Result<::Item>; 58 | fn next(&mut self) -> Option { 59 | match self.0 { 60 | OsmObjsImpl::OkIter(ref mut iter) => iter.next(), 61 | OsmObjsImpl::ErrIter(ref mut iter) => iter.next(), 62 | } 63 | } 64 | } 65 | 66 | /// Transforms a `Result` into a `Iterator>`. 67 | pub fn result_blob_into_iter(result: crate::Result) -> OsmObjs { 68 | match result.and_then(|b| crate::reader::primitive_block_from_blob(&b)) { 69 | Ok(block) => OsmObjs(OsmObjsImpl::OkIter( 70 | OsmBlobObjs::new(block, blocks::iter).map(Ok), 71 | )), 72 | Err(e) => OsmObjs(OsmObjsImpl::ErrIter(iter::once(Err(e)))), 73 | } 74 | } 75 | 76 | /// Transforms a `Result` into a `Iterator>`. 77 | pub fn result_blob_into_node_iter(result: crate::Result) -> OsmObjs { 78 | match result.and_then(|b| crate::reader::primitive_block_from_blob(&b)) { 79 | Ok(block) => OsmObjs(OsmObjsImpl::OkIter( 80 | OsmBlobNodes::new(block, blocks::nodes).map(Ok), 81 | )), 82 | Err(e) => OsmObjs(OsmObjsImpl::ErrIter(iter::once(Err(e)))), 83 | } 84 | } 85 | 86 | /// Transforms a `Result` into a `Iterator>`. 87 | pub fn result_blob_into_way_iter(result: crate::Result) -> OsmObjs { 88 | match result.and_then(|b| crate::reader::primitive_block_from_blob(&b)) { 89 | Ok(block) => OsmObjs(OsmObjsImpl::OkIter( 90 | OsmBlobWays::new(block, blocks::ways).map(Ok), 91 | )), 92 | Err(e) => OsmObjs(OsmObjsImpl::ErrIter(iter::once(Err(e)))), 93 | } 94 | } 95 | 96 | /// Transforms a `Result` into a `Iterator>`. 97 | pub fn result_blob_into_relation_iter(result: crate::Result) -> OsmObjs { 98 | match result.and_then(|b| crate::reader::primitive_block_from_blob(&b)) { 99 | Ok(block) => OsmObjs(OsmObjsImpl::OkIter( 100 | OsmBlobRelations::new(block, blocks::relations).map(Ok), 101 | )), 102 | Err(e) => OsmObjs(OsmObjsImpl::ErrIter(iter::once(Err(e)))), 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /src/blocks.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014-2015 Guillaume Pinot 2 | // 3 | // This work is free. You can redistribute it and/or modify it under 4 | // the terms of the Do What The Fuck You Want To Public License, 5 | // Version 2, as published by Sam Hocevar. See the COPYING file for 6 | // more details. 7 | 8 | //! Iterators of OpenStreetMap objects from a block. 9 | 10 | use crate::groups; 11 | use crate::objects::{Node, OsmObj, Relation, Way}; 12 | use crate::osmformat::PrimitiveBlock; 13 | use pub_iterator_type::pub_iterator_type; 14 | 15 | pub_iterator_type! { 16 | #[doc="Iterator on the `OsmObj` of a `PrimitiveBlock`."] 17 | OsmObjs['a] = Box + 'a + Send> 18 | } 19 | 20 | pub fn iter(block: &PrimitiveBlock) -> OsmObjs { 21 | let f = move |g| groups::iter(g, block); 22 | OsmObjs(Box::new(block.primitivegroup.iter().flat_map(f))) 23 | } 24 | 25 | pub_iterator_type! { 26 | #[doc="Iterator on the `Node` of a `PrimitiveBlock`."] 27 | Nodes['a] = Box + 'a + Send> 28 | } 29 | 30 | pub fn nodes(block: &PrimitiveBlock) -> Nodes { 31 | let f = move |g| groups::nodes(g, block); 32 | Nodes(Box::new(block.primitivegroup.iter().flat_map(f))) 33 | } 34 | 35 | pub_iterator_type! { 36 | #[doc="Iterator on the `Way` of a `PrimitiveBlock`."] 37 | Ways['a] = Box + 'a + Send> 38 | } 39 | 40 | pub fn ways(block: &PrimitiveBlock) -> Ways { 41 | let f = move |g| groups::ways(g, block); 42 | Ways(Box::new(block.primitivegroup.iter().flat_map(f))) 43 | } 44 | 45 | pub_iterator_type! { 46 | #[doc="Iterator on the `Relation` of a `PrimitiveBlock`."] 47 | Relations['a] = Box + 'a + Send> 48 | } 49 | 50 | pub fn relations(block: &PrimitiveBlock) -> Relations { 51 | let f = move |g| groups::relations(g, block); 52 | Relations(Box::new(block.primitivegroup.iter().flat_map(f))) 53 | } 54 | -------------------------------------------------------------------------------- /src/error.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014-2015 Guillaume Pinot 2 | // 3 | // This work is free. You can redistribute it and/or modify it under 4 | // the terms of the Do What The Fuck You Want To Public License, 5 | // Version 2, as published by Sam Hocevar. See the COPYING file for 6 | // more details. 7 | 8 | use protobuf; 9 | use std::convert::From; 10 | use std::{self, fmt, io}; 11 | 12 | pub type Result = std::result::Result; 13 | 14 | #[derive(Debug)] 15 | pub enum Error { 16 | Io(io::Error), 17 | Pbf(protobuf::Error), 18 | UnsupportedData, 19 | InvalidData, 20 | } 21 | 22 | impl fmt::Display for Error { 23 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 24 | match *self { 25 | Error::Io(ref e) => write!(f, "Io({})", e), 26 | Error::Pbf(ref e) => write!(f, "Pbf({})", e), 27 | Error::UnsupportedData => write!(f, "UnsupportedData"), 28 | Error::InvalidData => write!(f, "InvalidData"), 29 | } 30 | } 31 | } 32 | 33 | impl std::error::Error for Error { 34 | fn cause(&self) -> Option<&dyn std::error::Error> { 35 | match *self { 36 | Error::Io(ref e) => Some(e), 37 | Error::Pbf(ref e) => Some(e), 38 | Error::UnsupportedData | Error::InvalidData => None, 39 | } 40 | } 41 | } 42 | 43 | impl From for Error { 44 | fn from(err: io::Error) -> Error { 45 | Error::Io(err) 46 | } 47 | } 48 | 49 | impl From for Error { 50 | fn from(err: protobuf::Error) -> Error { 51 | Error::Pbf(err) 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/groups.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014-2015 Guillaume Pinot 2 | // 3 | // This work is free. You can redistribute it and/or modify it under 4 | // the terms of the Do What The Fuck You Want To Public License, 5 | // Version 2, as published by Sam Hocevar. See the COPYING file for 6 | // more details. 7 | 8 | use crate::objects::*; 9 | use crate::osmformat; 10 | use crate::osmformat::{PrimitiveBlock, PrimitiveGroup}; 11 | use pub_iterator_type::pub_iterator_type; 12 | use smartstring::alias::String; 13 | use std::borrow::Cow; 14 | use std::convert::From; 15 | use std::iter::Chain; 16 | use std::iter::Map; 17 | use std::slice; 18 | 19 | pub_iterator_type! { 20 | #[doc="Iterator on the `OsmObj` of a `PrimitiveGroup`."] 21 | OsmObjs['a] = Chain, fn(Node) -> OsmObj>, 22 | Map, fn(Way) -> OsmObj>>, 23 | Map, fn(Relation) -> OsmObj>> 24 | } 25 | 26 | pub fn iter<'a>(g: &'a PrimitiveGroup, b: &'a PrimitiveBlock) -> OsmObjs<'a> { 27 | let iter = nodes(g, b) 28 | .map(From::from as fn(Node) -> OsmObj) 29 | .chain(ways(g, b).map(From::from as fn(Way) -> OsmObj)) 30 | .chain(relations(g, b).map(From::from as fn(Relation) -> OsmObj)); 31 | OsmObjs(iter) 32 | } 33 | 34 | pub_iterator_type! { 35 | #[doc="Iterator on the `Node` of a `PrimitiveGroup`."] 36 | Nodes['a] = Chain, DenseNodes<'a>> 37 | } 38 | 39 | pub fn nodes<'a>(g: &'a PrimitiveGroup, b: &'a PrimitiveBlock) -> Nodes<'a> { 40 | Nodes(simple_nodes(g, b).chain(dense_nodes(g, b))) 41 | } 42 | 43 | pub fn simple_nodes<'a>(group: &'a PrimitiveGroup, block: &'a PrimitiveBlock) -> SimpleNodes<'a> { 44 | SimpleNodes { 45 | iter: group.nodes.iter(), 46 | block, 47 | } 48 | } 49 | 50 | pub struct SimpleNodes<'a> { 51 | iter: slice::Iter<'a, osmformat::Node>, 52 | block: &'a PrimitiveBlock, 53 | } 54 | 55 | impl Iterator for SimpleNodes<'_> { 56 | type Item = Node; 57 | fn next(&mut self) -> Option { 58 | self.iter.next().map(|n| Node { 59 | id: NodeId(n.id()), 60 | decimicro_lat: make_lat(n.lat(), self.block), 61 | decimicro_lon: make_lon(n.lon(), self.block), 62 | tags: make_tags(&n.keys, &n.vals, self.block), 63 | }) 64 | } 65 | fn size_hint(&self) -> (usize, Option) { 66 | self.iter.size_hint() 67 | } 68 | } 69 | 70 | pub fn dense_nodes<'a>(group: &'a PrimitiveGroup, block: &'a PrimitiveBlock) -> DenseNodes<'a> { 71 | let dense = &group.dense; 72 | DenseNodes { 73 | block, 74 | dids: dense.id.iter(), 75 | dlats: dense.lat.iter(), 76 | dlons: dense.lon.iter(), 77 | keys_vals: dense.keys_vals.iter(), 78 | cur_id: 0, 79 | cur_lat: 0, 80 | cur_lon: 0, 81 | } 82 | } 83 | 84 | pub struct DenseNodes<'a> { 85 | block: &'a PrimitiveBlock, 86 | dids: slice::Iter<'a, i64>, 87 | dlats: slice::Iter<'a, i64>, 88 | dlons: slice::Iter<'a, i64>, 89 | keys_vals: slice::Iter<'a, i32>, 90 | cur_id: i64, 91 | cur_lat: i64, 92 | cur_lon: i64, 93 | } 94 | 95 | impl Iterator for DenseNodes<'_> { 96 | type Item = Node; 97 | fn next(&mut self) -> Option { 98 | match (self.dids.next(), self.dlats.next(), self.dlons.next()) { 99 | (Some(&did), Some(&dlat), Some(&dlon)) => { 100 | self.cur_id += did; 101 | self.cur_lat += dlat; 102 | self.cur_lon += dlon; 103 | } 104 | _ => return None, 105 | } 106 | let mut tags = Tags::new(); 107 | loop { 108 | let k = match self.keys_vals.next() { 109 | None | Some(&0) => break, 110 | Some(k) => make_string(*k as usize, self.block), 111 | }; 112 | let v = match self.keys_vals.next() { 113 | None => break, 114 | Some(v) => make_string(*v as usize, self.block), 115 | }; 116 | tags.insert(k, v); 117 | } 118 | tags.shrink_to_fit(); 119 | Some(Node { 120 | id: NodeId(self.cur_id), 121 | decimicro_lat: make_lat(self.cur_lat, self.block), 122 | decimicro_lon: make_lon(self.cur_lon, self.block), 123 | tags, 124 | }) 125 | } 126 | } 127 | 128 | pub fn ways<'a>(group: &'a PrimitiveGroup, block: &'a PrimitiveBlock) -> Ways<'a> { 129 | Ways { 130 | iter: group.ways.iter(), 131 | block, 132 | } 133 | } 134 | 135 | pub struct Ways<'a> { 136 | iter: slice::Iter<'a, osmformat::Way>, 137 | block: &'a PrimitiveBlock, 138 | } 139 | 140 | impl Iterator for Ways<'_> { 141 | type Item = Way; 142 | fn next(&mut self) -> Option { 143 | self.iter.next().map(|w| { 144 | let mut n = 0; 145 | let nodes = w 146 | .refs 147 | .iter() 148 | .map(|&dn| { 149 | n += dn; 150 | NodeId(n) 151 | }) 152 | .collect(); 153 | Way { 154 | id: WayId(w.id()), 155 | nodes, 156 | tags: make_tags(&w.keys, &w.vals, self.block), 157 | } 158 | }) 159 | } 160 | fn size_hint(&self) -> (usize, Option) { 161 | self.iter.size_hint() 162 | } 163 | } 164 | 165 | pub fn relations<'a>(group: &'a PrimitiveGroup, block: &'a PrimitiveBlock) -> Relations<'a> { 166 | Relations { 167 | iter: group.relations.iter(), 168 | block, 169 | } 170 | } 171 | pub struct Relations<'a> { 172 | iter: slice::Iter<'a, osmformat::Relation>, 173 | block: &'a PrimitiveBlock, 174 | } 175 | 176 | impl Iterator for Relations<'_> { 177 | type Item = Relation; 178 | fn next(&mut self) -> Option { 179 | use osmformat::relation::MemberType::{NODE, RELATION, WAY}; 180 | self.iter.next().map(|rel| { 181 | let mut m = 0; 182 | let refs = rel 183 | .memids 184 | .iter() 185 | .zip(rel.types.iter()) 186 | .zip(rel.roles_sid.iter()) 187 | .flat_map(|((&dm, &t), &role)| { 188 | m += dm; 189 | t.enum_value().map(|t| Ref { 190 | member: match t { 191 | NODE => NodeId(m).into(), 192 | WAY => WayId(m).into(), 193 | RELATION => RelationId(m).into(), 194 | }, 195 | role: make_string(role as usize, self.block), 196 | }) 197 | }) 198 | .collect(); 199 | Relation { 200 | id: RelationId(rel.id()), 201 | refs, 202 | tags: make_tags(&rel.keys, &rel.vals, self.block), 203 | } 204 | }) 205 | } 206 | fn size_hint(&self) -> (usize, Option) { 207 | self.iter.size_hint() 208 | } 209 | } 210 | 211 | fn make_string(k: usize, block: &osmformat::PrimitiveBlock) -> String { 212 | let cow = std::string::String::from_utf8_lossy(&block.stringtable.s[k]); 213 | match cow { 214 | Cow::Borrowed(s) => String::from(s), 215 | Cow::Owned(s) => String::from(s), 216 | } 217 | } 218 | 219 | fn make_lat(c: i64, b: &osmformat::PrimitiveBlock) -> i32 { 220 | let granularity = b.granularity() as i64; 221 | ((b.lat_offset() + granularity * c) / 100) as i32 222 | } 223 | 224 | fn make_lon(c: i64, b: &osmformat::PrimitiveBlock) -> i32 { 225 | let granularity = b.granularity() as i64; 226 | ((b.lon_offset() + granularity * c) / 100) as i32 227 | } 228 | 229 | fn make_tags(keys: &[u32], vals: &[u32], b: &PrimitiveBlock) -> Tags { 230 | let mut tags: Tags = keys 231 | .iter() 232 | .zip(vals.iter()) 233 | .map(|(&k, &v)| (make_string(k as usize, b), make_string(v as usize, b))) 234 | .collect(); 235 | tags.shrink_to_fit(); 236 | tags 237 | } 238 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014-2015 Guillaume Pinot 2 | // 3 | // This work is free. You can redistribute it and/or modify it under 4 | // the terms of the Do What The Fuck You Want To Public License, 5 | // Version 2, as published by Sam Hocevar. See the COPYING file for 6 | // more details. 7 | 8 | //! This crate provide an interface to easily read [OpenStreetMap PBF 9 | //! files](http://wiki.openstreetmap.org/wiki/PBF_Format). Its main 10 | //! inspiration is 11 | //! [libosmpbfreader](https://github.com/CanalTP/libosmpbfreader). 12 | //! 13 | //! # Getting objects and their dependencies 14 | //! 15 | //! Most of the time, you'll want a subset of the OSM objects and its 16 | //! dependencies (i.e. the nodes inside a way, and not only the ids of 17 | //! the nodes of this way). For that, an easy to use function is 18 | //! availlable. 19 | //! 20 | //! ```rust 21 | //! let mut pbf = osmpbfreader::OsmPbfReader::new(std::io::Cursor::new([])); 22 | //! let objs = pbf.get_objs_and_deps(|obj| { 23 | //! obj.is_way() && obj.tags().contains_key("highway") 24 | //! }) 25 | //! .unwrap(); 26 | //! for (id, obj) in &objs { 27 | //! println!("{:?}: {:?}", id, obj); 28 | //! } 29 | //! ``` 30 | //! 31 | //! # Reading 32 | //! 33 | //! The easiest way to read a PBF file is to directly iterate on the 34 | //! `OsmObj`. 35 | //! 36 | //! ```rust 37 | //! use std::process::exit; 38 | //! let mut pbf = osmpbfreader::OsmPbfReader::new(std::io::empty()); 39 | //! for obj in pbf.iter() { 40 | //! // error handling: 41 | //! let obj = obj.unwrap_or_else(|e| {println!("{:?}", e); exit(1)}); 42 | //! 43 | //! println!("{:?}", obj); 44 | //! } 45 | //! ``` 46 | //! 47 | //! There is also a parallel version of this iterator. The file is 48 | //! decoded in parallel. 49 | //! 50 | //! ```rust 51 | //! use std::process::exit; 52 | //! let mut pbf = osmpbfreader::OsmPbfReader::new(std::io::empty()); 53 | //! for obj in pbf.par_iter() { 54 | //! // error handling: 55 | //! let obj = obj.unwrap_or_else(|e| {println!("{:?}", e); exit(1)}); 56 | //! 57 | //! println!("{:?}", obj); 58 | //! } 59 | //! ``` 60 | //! 61 | //! # Into the details 62 | //! 63 | //! This crate is build around basic iterators on different parts of 64 | //! the structure of the PBF format. Then, several higher level 65 | //! iterator are proposed. It is then possible to iterate on the file 66 | //! using the low level iterators. 67 | //! 68 | //! ```rust 69 | //! use osmpbfreader::{primitive_block_from_blob, groups}; 70 | //! let mut pbf = osmpbfreader::OsmPbfReader::new(std::io::empty()); 71 | //! for block in pbf.blobs().map(|b| primitive_block_from_blob(&b.unwrap())) { 72 | //! let block = block.unwrap(); 73 | //! for group in block.primitivegroup.iter() { 74 | //! for node in groups::simple_nodes(&group, &block) { 75 | //! println!("{:?}", node); 76 | //! } 77 | //! for node in groups::dense_nodes(&group, &block) { 78 | //! println!("{:?}", node); 79 | //! } 80 | //! for way in groups::ways(&group, &block) { 81 | //! println!("{:?}", way); 82 | //! } 83 | //! for relation in groups::relations(&group, &block) { 84 | //! println!("{:?}", relation); 85 | //! } 86 | //! } 87 | //! } 88 | //! ``` 89 | //! 90 | //! Notice that `primitive_block_from_blob` can be costy as it 91 | //! uncompress the blob. Using some kind of parallel map can then 92 | //! improve the reading speed of the PBF file. 93 | //! 94 | //! OSM tags are stored as key/value maps of type `smartstring::alias::String`. [smartstring](https://crates.io/crates/smartstring) is a crate which re-implements the `std::str::String` API, while avoiding heap allocation for short strings (up to 23 bytes on 64 bit architectures). Longer strings are allocated transparently. Most OSM tags keys & values are rather short, so using this type can yield substantial improvements for performance and memory usage. 95 | 96 | #![deny(missing_docs)] 97 | 98 | pub use error::Error; 99 | pub use error::Result; 100 | pub use objects::*; 101 | pub use reader::{primitive_block_from_blob, OsmPbfReader, StoreObjs}; 102 | 103 | pub mod blobs; 104 | pub mod objects; 105 | pub mod reader; 106 | 107 | #[allow(missing_docs)] 108 | pub mod blocks; 109 | #[allow(missing_docs)] 110 | pub mod error; 111 | #[allow(missing_docs)] 112 | pub mod groups; 113 | 114 | mod pbf { 115 | include!(concat!(env!("OUT_DIR"), "/mod.rs")); 116 | } 117 | 118 | pub use pbf::{fileformat, osmformat}; 119 | 120 | /* 121 | /// Generated from protobuf. 122 | #[allow(non_snake_case, missing_docs)] 123 | pub mod fileformat; 124 | 125 | /// Generated from protobuf. 126 | #[allow(missing_docs)] 127 | pub mod osmformat; 128 | */ 129 | -------------------------------------------------------------------------------- /src/objects.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014-2015 Guillaume Pinot 2 | // 3 | // This work is free. You can redistribute it and/or modify it under 4 | // the terms of the Do What The Fuck You Want To Public License, 5 | // Version 2, as published by Sam Hocevar. See the COPYING file for 6 | // more details. 7 | 8 | //! This module proposes objects to modelize OpenStreetMap objects. 9 | //! 10 | //! There are 3 types of objects: nodes, ways and relations. 11 | 12 | #[cfg(feature = "serde")] 13 | use serde::{Deserialize, Serialize}; 14 | use smartstring::alias::String; 15 | use std::iter::FromIterator; 16 | use std::ops::{Deref, DerefMut}; 17 | 18 | /// Tags represents the features of the objects. See the 19 | /// [OpenStreetMap wiki page about 20 | /// tags](http://wiki.openstreetmap.org/wiki/Tags) for more 21 | /// information. 22 | #[derive(Debug, Default, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] 23 | #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] 24 | pub struct Tags(::flat_map::FlatMap); 25 | 26 | impl Tags { 27 | /// Creates a new, empty `Tags` object. 28 | pub fn new() -> Tags { 29 | Tags(::flat_map::FlatMap::new()) 30 | } 31 | 32 | /// Returns if it contains a tag with the given `key` and `value`. 33 | pub fn contains(&self, key: &str, value: &str) -> bool { 34 | self.0.get(key).is_some_and(|v| v.as_str() == value) 35 | } 36 | 37 | /// Consume tags into inner FlatMap representation 38 | pub fn into_inner(self) -> ::flat_map::FlatMap { 39 | self.0 40 | } 41 | } 42 | 43 | impl Deref for Tags { 44 | type Target = ::flat_map::FlatMap; 45 | 46 | fn deref(&self) -> &Self::Target { 47 | &self.0 48 | } 49 | } 50 | 51 | impl DerefMut for Tags { 52 | fn deref_mut(&mut self) -> &mut Self::Target { 53 | &mut self.0 54 | } 55 | } 56 | 57 | impl FromIterator<(String, String)> for Tags { 58 | fn from_iter>(iter: T) -> Self { 59 | Tags(iter.into_iter().collect()) 60 | } 61 | } 62 | 63 | /// A node identifier 64 | #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Hash, Copy)] 65 | #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] 66 | pub struct NodeId(pub i64); 67 | 68 | /// A way identifier 69 | #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Hash, Copy)] 70 | #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] 71 | pub struct WayId(pub i64); 72 | 73 | /// A relation identifier 74 | #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Hash, Copy)] 75 | #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] 76 | pub struct RelationId(pub i64); 77 | 78 | /// An OpenStreetMap object identifier 79 | #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Hash, Copy)] 80 | #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] 81 | pub enum OsmId { 82 | /// The identifier of a node 83 | Node(NodeId), 84 | /// The identifier of a way 85 | Way(WayId), 86 | /// The identifier of a relation 87 | Relation(RelationId), 88 | } 89 | 90 | impl OsmId { 91 | /// Returns `true` if the id is a node id. 92 | pub fn is_node(&self) -> bool { 93 | self.node().is_some() 94 | } 95 | /// Returns `true` if the id is a way id. 96 | pub fn is_way(&self) -> bool { 97 | self.way().is_some() 98 | } 99 | /// Returns `true` if the id is a relation id. 100 | pub fn is_relation(&self) -> bool { 101 | self.relation().is_some() 102 | } 103 | /// Returns the `NodeId` if it is a node, otherwise returns `None`. 104 | pub fn node(&self) -> Option { 105 | match *self { 106 | OsmId::Node(id) => Some(id), 107 | _ => None, 108 | } 109 | } 110 | /// Returns the `WayId` if it is a way, otherwise returns `None`. 111 | pub fn way(&self) -> Option { 112 | match *self { 113 | OsmId::Way(id) => Some(id), 114 | _ => None, 115 | } 116 | } 117 | /// Returns the `RelationId` if it is a relation, otherwise returns `None`. 118 | pub fn relation(&self) -> Option { 119 | match *self { 120 | OsmId::Relation(id) => Some(id), 121 | _ => None, 122 | } 123 | } 124 | /// Returns the inner id. 125 | pub fn inner_id(&self) -> i64 { 126 | match *self { 127 | OsmId::Node(n) => n.0, 128 | OsmId::Way(n) => n.0, 129 | OsmId::Relation(n) => n.0, 130 | } 131 | } 132 | } 133 | 134 | /// An OpenStreetMap object. 135 | #[derive(Debug, PartialEq, PartialOrd, Clone)] 136 | #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] 137 | pub enum OsmObj { 138 | /// A node 139 | Node(Node), 140 | /// A way 141 | Way(Way), 142 | /// A relation 143 | Relation(Relation), 144 | } 145 | 146 | impl OsmObj { 147 | /// Returns the tags of the object. 148 | pub fn tags(&self) -> &Tags { 149 | match *self { 150 | OsmObj::Node(ref node) => &node.tags, 151 | OsmObj::Way(ref way) => &way.tags, 152 | OsmObj::Relation(ref rel) => &rel.tags, 153 | } 154 | } 155 | /// Returns the id of the object. 156 | pub fn id(&self) -> OsmId { 157 | match *self { 158 | OsmObj::Node(ref node) => OsmId::Node(node.id), 159 | OsmObj::Way(ref way) => OsmId::Way(way.id), 160 | OsmObj::Relation(ref rel) => OsmId::Relation(rel.id), 161 | } 162 | } 163 | /// Returns `true` if the object is a node. 164 | pub fn is_node(&self) -> bool { 165 | self.node().is_some() 166 | } 167 | /// Returns `true` if the object is a way. 168 | pub fn is_way(&self) -> bool { 169 | self.way().is_some() 170 | } 171 | /// Returns `true` if the object is a relation. 172 | pub fn is_relation(&self) -> bool { 173 | self.relation().is_some() 174 | } 175 | /// Returns a reference to the `Node` if `self` is a node, otherwise returns `None`. 176 | pub fn node(&self) -> Option<&Node> { 177 | match *self { 178 | OsmObj::Node(ref n) => Some(n), 179 | _ => None, 180 | } 181 | } 182 | /// Returns a reference to the `Way` if `self` is a way, otherwise returns `None`. 183 | pub fn way(&self) -> Option<&Way> { 184 | match *self { 185 | OsmObj::Way(ref w) => Some(w), 186 | _ => None, 187 | } 188 | } 189 | /// Returns a reference to the `Relation` if `self` is a relation, otherwise returns `None`. 190 | pub fn relation(&self) -> Option<&Relation> { 191 | match *self { 192 | OsmObj::Relation(ref r) => Some(r), 193 | _ => None, 194 | } 195 | } 196 | } 197 | 198 | /// An OpenStreetMap node. See the [OpenStreetMap wiki page about 199 | /// node](http://wiki.openstreetmap.org/wiki/Node) for more 200 | /// information. 201 | #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Hash)] 202 | #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] 203 | pub struct Node { 204 | /// The id of the node. 205 | pub id: NodeId, 206 | /// The tags of the node. 207 | pub tags: Tags, 208 | /// The latitude in decimicro degrees (10⁻⁷ degrees). 209 | pub decimicro_lat: i32, 210 | /// The longitude in decimicro degrees (10⁻⁷ degrees). 211 | pub decimicro_lon: i32, 212 | } 213 | 214 | impl Node { 215 | /// Returns the latitude of the node in degrees. 216 | pub fn lat(&self) -> f64 { 217 | self.decimicro_lat as f64 * 1e-7 218 | } 219 | /// Returns the longitude of the node in degrees. 220 | pub fn lon(&self) -> f64 { 221 | self.decimicro_lon as f64 * 1e-7 222 | } 223 | } 224 | 225 | /// An OpenStreetMap way. See the [OpenStreetMap wiki page about 226 | /// way](http://wiki.openstreetmap.org/wiki/Way) for more 227 | /// information. 228 | #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Hash)] 229 | #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] 230 | pub struct Way { 231 | /// The id of the way. 232 | pub id: WayId, 233 | /// The tags of the way. 234 | pub tags: Tags, 235 | /// The ordered list of nodes as id. 236 | pub nodes: Vec, 237 | } 238 | 239 | impl Way { 240 | /// Returns true if the way is 241 | /// [open](http://wiki.openstreetmap.org/wiki/Way#Open_way). 242 | pub fn is_open(&self) -> bool { 243 | !self.is_closed() 244 | } 245 | /// Returns true if the way is 246 | /// [closed](http://wiki.openstreetmap.org/wiki/Way#Closed_way). 247 | pub fn is_closed(&self) -> bool { 248 | self.nodes.first() == self.nodes.last() 249 | } 250 | } 251 | 252 | /// A reference to an object with a role. Used in the relation object. 253 | #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Hash)] 254 | #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] 255 | pub struct Ref { 256 | /// Id of the member. 257 | pub member: OsmId, 258 | /// Role of the member. 259 | pub role: String, 260 | } 261 | 262 | /// An OpenStreetMap relation. See the [OpenStreetMap wiki page about 263 | /// relation](http://wiki.openstreetmap.org/wiki/Relation) for more 264 | /// information. 265 | #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Hash)] 266 | #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] 267 | pub struct Relation { 268 | /// The id of the relation. 269 | pub id: RelationId, 270 | /// The tags of the relation. 271 | pub tags: Tags, 272 | /// Members of the relation. 273 | pub refs: Vec, 274 | } 275 | 276 | impl ::std::convert::From for OsmId { 277 | fn from(n: NodeId) -> Self { 278 | OsmId::Node(n) 279 | } 280 | } 281 | 282 | impl ::std::convert::From for OsmId { 283 | fn from(w: WayId) -> Self { 284 | OsmId::Way(w) 285 | } 286 | } 287 | 288 | impl ::std::convert::From for OsmId { 289 | fn from(r: RelationId) -> Self { 290 | OsmId::Relation(r) 291 | } 292 | } 293 | 294 | impl ::std::convert::From for OsmObj { 295 | fn from(n: Node) -> Self { 296 | OsmObj::Node(n) 297 | } 298 | } 299 | 300 | impl ::std::convert::From for OsmObj { 301 | fn from(w: Way) -> Self { 302 | OsmObj::Way(w) 303 | } 304 | } 305 | 306 | impl ::std::convert::From for OsmObj { 307 | fn from(r: Relation) -> Self { 308 | OsmObj::Relation(r) 309 | } 310 | } 311 | -------------------------------------------------------------------------------- /src/reader.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014-2017 Guillaume Pinot 2 | // 3 | // This work is free. You can redistribute it and/or modify it under 4 | // the terms of the Do What The Fuck You Want To Public License, 5 | // Version 2, as published by Sam Hocevar. See the COPYING file for 6 | // more details. 7 | 8 | //! Tools for reading a pbf file. 9 | 10 | use crate::blobs; 11 | use crate::error::{Error, Result}; 12 | use crate::fileformat::{Blob, BlobHeader}; 13 | use crate::objects::{OsmId, OsmObj}; 14 | use crate::osmformat::PrimitiveBlock; 15 | use par_map::{self, ParMap}; 16 | use protobuf::Message; 17 | use pub_iterator_type::pub_iterator_type; 18 | use std::collections::btree_map::BTreeMap; 19 | use std::collections::BTreeSet; 20 | use std::convert::From; 21 | use std::io::{self, Read}; 22 | use std::iter; 23 | 24 | /// Trait to allow generic objects (not just BTreeMap) in some methods. 25 | pub trait StoreObjs { 26 | /// Insert given object at given key index. 27 | fn insert(&mut self, key: OsmId, value: OsmObj); 28 | /// Check if object contains the given key. 29 | fn contains_key(&self, key: &OsmId) -> bool; 30 | } 31 | 32 | impl StoreObjs for BTreeMap { 33 | fn insert(&mut self, key: OsmId, value: OsmObj) { 34 | self.insert(key, value); 35 | } 36 | 37 | fn contains_key(&self, key: &OsmId) -> bool { 38 | self.contains_key(key) 39 | } 40 | } 41 | 42 | /// The object to manage a pbf file. 43 | pub struct OsmPbfReader { 44 | buf: Vec, 45 | r: R, 46 | finished: bool, 47 | } 48 | 49 | impl OsmPbfReader { 50 | /// Creates an OsmPbfReader from a Read object. 51 | pub fn new(r: R) -> OsmPbfReader { 52 | OsmPbfReader { 53 | buf: vec![], 54 | r, 55 | finished: false, 56 | } 57 | } 58 | 59 | /// Returns an iterator on the OsmObj of the pbf file. 60 | /// 61 | /// # Example 62 | /// 63 | /// ``` 64 | /// let mut pbf = osmpbfreader::OsmPbfReader::new(std::io::empty()); 65 | /// for obj in pbf.iter().map(Result::unwrap) { 66 | /// println!("{:?}", obj); 67 | /// } 68 | /// ``` 69 | pub fn iter(&mut self) -> Iter { 70 | Iter(self.blobs().flat_map(blobs::result_blob_into_iter)) 71 | } 72 | 73 | /// Returns a parallel iterator on the OsmObj of the pbf file. 74 | /// 75 | /// Several threads decode in parallel the file. The memory and 76 | /// CPU usage are guaranteed to be bounded even if the caller stop 77 | /// consuming items. 78 | /// 79 | /// # Example 80 | /// 81 | /// ``` 82 | /// let mut pbf = osmpbfreader::OsmPbfReader::new(std::io::empty()); 83 | /// for obj in pbf.par_iter().map(Result::unwrap) { 84 | /// println!("{:?}", obj); 85 | /// } 86 | /// ``` 87 | pub fn par_iter(&mut self) -> ParIter<'_, R> { 88 | ParIter(self.blobs().par_flat_map(blobs::result_blob_into_iter)) 89 | } 90 | 91 | /// Returns an iterator on the Node of the pbf file. 92 | pub fn iter_nodes(&mut self) -> NodeIter { 93 | NodeIter(self.blobs().flat_map(blobs::result_blob_into_node_iter)) 94 | } 95 | 96 | /// Returns a parallel iterator on the Node of the pbf file. 97 | /// 98 | /// Several threads decode in parallel the file. The memory and 99 | /// CPU usage are guaranteed to be bounded even if the caller stop 100 | /// consuming items. 101 | pub fn par_iter_nodes(&mut self) -> NodeParIter<'_, R> { 102 | NodeParIter(self.blobs().par_flat_map(blobs::result_blob_into_node_iter)) 103 | } 104 | 105 | /// Returns an iterator on the Way of the pbf file. 106 | pub fn iter_ways(&mut self) -> WayIter { 107 | WayIter(self.blobs().flat_map(blobs::result_blob_into_way_iter)) 108 | } 109 | 110 | /// Returns a parallel iterator on the Way of the pbf file. 111 | /// 112 | /// Several threads decode in parallel the file. The memory and 113 | /// CPU usage are guaranteed to be bounded even if the caller stop 114 | /// consuming items. 115 | pub fn par_iter_ways(&mut self) -> WayParIter<'_, R> { 116 | WayParIter(self.blobs().par_flat_map(blobs::result_blob_into_way_iter)) 117 | } 118 | 119 | /// Returns an iterator on the Relation of the pbf file. 120 | pub fn iter_relations(&mut self) -> RelationIter { 121 | RelationIter(self.blobs().flat_map(blobs::result_blob_into_relation_iter)) 122 | } 123 | 124 | /// Returns a parallel iterator on the Relation of the pbf file. 125 | /// 126 | /// Several threads decode in parallel the file. The memory and 127 | /// CPU usage are guaranteed to be bounded even if the caller stop 128 | /// consuming items. 129 | pub fn par_iter_relations(&mut self) -> RelationParIter<'_, R> { 130 | RelationParIter( 131 | self.blobs() 132 | .par_flat_map(blobs::result_blob_into_relation_iter), 133 | ) 134 | } 135 | 136 | /// Rewinds the pbf file to the begining. 137 | /// 138 | /// Useful if you want to read several consecutive times the same 139 | /// file. 140 | /// 141 | /// # Example 142 | /// 143 | /// ``` 144 | /// let mut cursor = std::io::Cursor::new([0, 0, 0]); 145 | /// cursor.set_position(2); 146 | /// let mut pbf = osmpbfreader::OsmPbfReader::new(cursor); 147 | /// pbf.rewind().unwrap(); 148 | /// assert_eq!(pbf.into_inner().position(), 0); 149 | /// ``` 150 | pub fn rewind(&mut self) -> Result<()> 151 | where 152 | R: io::Seek, 153 | { 154 | self.r.seek(io::SeekFrom::Start(0))?; 155 | self.finished = false; 156 | Ok(()) 157 | } 158 | 159 | /// Same as `get_objs_and_deps` but generic. 160 | pub fn get_objs_and_deps_store(&mut self, mut pred: F, objects: &mut T) -> Result<()> 161 | where 162 | R: io::Seek, 163 | F: FnMut(&OsmObj) -> bool, 164 | T: StoreObjs, 165 | { 166 | let mut finished = false; 167 | let mut deps = BTreeSet::new(); 168 | let mut first_pass = true; 169 | while !finished { 170 | self.rewind()?; 171 | finished = true; 172 | for obj in self.par_iter() { 173 | let obj = obj?; 174 | if (!first_pass || !pred(&obj)) && !deps.contains(&obj.id()) { 175 | continue; 176 | } 177 | finished = match obj { 178 | OsmObj::Relation(ref rel) => rel 179 | .refs 180 | .iter() 181 | .filter(|r| !objects.contains_key(&r.member)) 182 | .fold(finished, |accu, r| !deps.insert(r.member) && accu), 183 | OsmObj::Way(ref way) => way 184 | .nodes 185 | .iter() 186 | .filter(|n| !objects.contains_key(&(**n).into())) 187 | .fold(finished, |accu, n| !deps.insert((*n).into()) && accu), 188 | OsmObj::Node(_) => finished, 189 | }; 190 | deps.remove(&obj.id()); 191 | objects.insert(obj.id(), obj); 192 | } 193 | first_pass = false; 194 | } 195 | Ok(()) 196 | } 197 | 198 | /// This function give you the ability to find all the objects 199 | /// validating a predicate and all their dependencies. The file 200 | /// will be decoded in parallel. 201 | /// 202 | /// # Example 203 | /// 204 | /// If you want to extract all the administrative boundaries 205 | /// and all their dependencies you can do something like that: 206 | /// 207 | /// ``` 208 | /// fn is_admin(obj: &osmpbfreader::OsmObj) -> bool { 209 | /// // get relations with tags[boundary] == administrative 210 | /// obj.is_relation() && obj.tags().contains("boundary", "administrative") 211 | /// } 212 | /// 213 | /// let mut pbf = osmpbfreader::OsmPbfReader::new(std::io::Cursor::new([])); 214 | /// let objs = pbf.get_objs_and_deps(is_admin).unwrap(); 215 | /// for (id, obj) in &objs { 216 | /// println!("{:?}: {:?}", id, obj); 217 | /// } 218 | /// ``` 219 | pub fn get_objs_and_deps(&mut self, pred: F) -> Result> 220 | where 221 | R: io::Seek, 222 | F: FnMut(&OsmObj) -> bool, 223 | { 224 | let mut objects = BTreeMap::new(); 225 | match self.get_objs_and_deps_store(pred, &mut objects) { 226 | Ok(_) => Ok(objects), 227 | Err(e) => Err(e), 228 | } 229 | } 230 | /// Extract the Read object. 231 | /// 232 | /// Consumes the object. 233 | pub fn into_inner(self) -> R { 234 | self.r 235 | } 236 | /// Returns an iterator on the blobs of the pbf file. 237 | pub fn blobs(&mut self) -> Blobs { 238 | Blobs { opr: self } 239 | } 240 | /// Returns an iterator on the blocks of the pbf file. 241 | pub fn primitive_blocks(&mut self) -> PrimitiveBlocks { 242 | fn and_then_primitive_block(blob_res: Result) -> Result { 243 | blob_res.and_then(|b| primitive_block_from_blob(&b)) 244 | } 245 | PrimitiveBlocks(self.blobs().map(and_then_primitive_block)) 246 | } 247 | 248 | fn push(&mut self, sz: u64) -> Result<()> { 249 | self.buf.clear(); 250 | self.r.by_ref().take(sz).read_to_end(&mut self.buf)?; 251 | assert_eq!(sz, self.buf.len() as u64); 252 | Ok(()) 253 | } 254 | fn try_blob(&mut self, sz: u64) -> Result> { 255 | self.push(sz)?; 256 | let header: BlobHeader = Message::parse_from_bytes(&self.buf)?; 257 | let sz = header.datasize() as u64; 258 | self.push(sz)?; 259 | let blob: Blob = Message::parse_from_bytes(&self.buf)?; 260 | if header.type_() == "OSMData" { 261 | Ok(Some(blob)) 262 | } else if header.type_() == "OSMHeader" { 263 | Ok(None) 264 | } else { 265 | println!("Unknown type: {}", header.type_()); 266 | Ok(None) 267 | } 268 | } 269 | fn next_blob(&mut self) -> Option> { 270 | use byteorder::{BigEndian, ReadBytesExt}; 271 | use std::io::ErrorKind; 272 | if self.finished { 273 | return None; 274 | } 275 | let sz = match self.r.read_u32::() { 276 | Ok(sz) if sz > 64 * 1024 => return Some(Err(Error::InvalidData)), 277 | Ok(sz) => sz, 278 | Err(ref e) if e.kind() == ErrorKind::UnexpectedEof => { 279 | self.finished = true; 280 | return None; 281 | } 282 | Err(e) => { 283 | self.finished = true; 284 | return Some(Err(From::from(e))); 285 | } 286 | } as u64; 287 | match self.try_blob(sz) { 288 | Ok(Some(p)) => Some(Ok(p)), 289 | Ok(None) => self.next_blob(), 290 | Err(e) => { 291 | self.finished = true; 292 | Some(Err(e)) 293 | } 294 | } 295 | } 296 | } 297 | 298 | /// Iterator on the blobs of a file. 299 | pub struct Blobs<'a, R: 'a> { 300 | opr: &'a mut OsmPbfReader, 301 | } 302 | impl Iterator for Blobs<'_, R> { 303 | type Item = Result; 304 | fn next(&mut self) -> Option { 305 | self.opr.next_blob() 306 | } 307 | } 308 | 309 | pub_iterator_type! { 310 | #[doc="Iterator on the blocks of a file."] 311 | PrimitiveBlocks['a, R] = iter::Map, fn(Result) -> Result> 312 | where R: Read + 'a 313 | } 314 | 315 | /// Returns an iterator on the blocks of a blob. 316 | pub fn primitive_block_from_blob(blob: &Blob) -> Result { 317 | if blob.has_raw() { 318 | Message::parse_from_bytes(blob.raw()).map_err(From::from) 319 | } else if blob.has_zlib_data() { 320 | use flate2::read::ZlibDecoder; 321 | let r = io::Cursor::new(blob.zlib_data()); 322 | let mut zr = ZlibDecoder::new(r); 323 | Message::parse_from_reader(&mut zr).map_err(From::from) 324 | } else { 325 | Err(Error::UnsupportedData) 326 | } 327 | } 328 | 329 | pub_iterator_type! { 330 | #[doc="Iterator on the `OsmObj` of the pbf file."] 331 | Iter['a, R] = iter::FlatMap, blobs::OsmObjs, fn(Result) -> blobs::OsmObjs> 332 | where R: io::Read + 'a 333 | } 334 | 335 | pub_iterator_type! { 336 | #[doc="Parallel iterator on the `OsmObj` of the pbf file."] 337 | ParIter['a, R] = par_map::FlatMap, 338 | blobs::OsmObjs, 339 | fn(Result) -> blobs::OsmObjs> 340 | where R: io::Read + 'a 341 | } 342 | 343 | pub_iterator_type! { 344 | #[doc="Iterator on the `OsmObj` of the pbf file."] 345 | NodeIter['a, R] = iter::FlatMap, blobs::OsmObjs, fn(Result) -> blobs::OsmObjs> 346 | where R: io::Read + 'a 347 | } 348 | 349 | pub_iterator_type! { 350 | #[doc="Parallel iterator on the `OsmObj` of the pbf file."] 351 | NodeParIter['a, R] = par_map::FlatMap, 352 | blobs::OsmObjs, 353 | fn(Result) -> blobs::OsmObjs> 354 | where R: io::Read + 'a 355 | } 356 | 357 | pub_iterator_type! { 358 | #[doc="Iterator on the `OsmObj` of the pbf file."] 359 | WayIter['a, R] = iter::FlatMap, blobs::OsmObjs, fn(Result) -> blobs::OsmObjs> 360 | where R: io::Read + 'a 361 | } 362 | 363 | pub_iterator_type! { 364 | #[doc="Parallel iterator on the `OsmObj` of the pbf file."] 365 | WayParIter['a, R] = par_map::FlatMap, 366 | blobs::OsmObjs, 367 | fn(Result) -> blobs::OsmObjs> 368 | where R: io::Read + 'a 369 | } 370 | 371 | pub_iterator_type! { 372 | #[doc="Iterator on the `OsmObj` of the pbf file."] 373 | RelationIter['a, R] = iter::FlatMap, blobs::OsmObjs, fn(Result) -> blobs::OsmObjs> 374 | where R: io::Read + 'a 375 | } 376 | 377 | pub_iterator_type! { 378 | #[doc="Parallel iterator on the `OsmObj` of the pbf file."] 379 | RelationParIter['a, R] = par_map::FlatMap, 380 | blobs::OsmObjs, 381 | fn(Result) -> blobs::OsmObjs> 382 | where R: io::Read + 'a 383 | } 384 | --------------------------------------------------------------------------------