├── .gitignore ├── .readthedocs.yml ├── CHANGELOG.md ├── LICENSE ├── MANIFEST.in ├── README.md ├── docs ├── Makefile ├── make.bat └── source │ ├── _images │ ├── GeometryCollection.png │ ├── GeometryCollection.svg │ ├── LineString.png │ ├── LineString.svg │ ├── MultiLineString.png │ ├── MultiLineString.svg │ ├── MultiPolygon.png │ ├── MultiPolygon.svg │ ├── Point.png │ ├── Point.svg │ ├── pbf_db_example.png │ ├── pbf_schemas_example.png │ ├── pbf_schemas_example_2.png │ └── pbf_table_example.png │ ├── _static │ ├── copy_button.css │ ├── prompt_button.js │ └── rtd_overrides.css │ ├── _templates │ ├── base.rst │ ├── breadcrumbs.html │ ├── class.rst │ ├── class2.rst │ ├── function.rst │ ├── latex.tex_t │ ├── longtable.tex_t │ └── module.rst │ ├── acknowledgement.rst │ ├── conf.py │ ├── contributors.rst │ ├── downloader.rst │ ├── errors.rst │ ├── index.rst │ ├── installation.rst │ ├── introduction.rst │ ├── ios.rst │ ├── latexindex.rst │ ├── license.rst │ ├── modules.rst │ ├── quick-start.rst │ ├── reader.rst │ ├── rtd-req.txt │ └── utils.rst ├── pydriosm ├── __init__.py ├── _updater.py ├── data │ ├── bbbike_cities.pkl │ ├── bbbike_cities_coordinates.pkl │ ├── bbbike_downloads_catalogue.pkl │ ├── bbbike_index_of_subregions.pkl │ ├── bbbike_subregion_names.pkl │ ├── geofabrik_continent_tables.pkl │ ├── geofabrik_downloads_catalogue.pkl │ ├── geofabrik_index_of_subregions.pkl │ ├── geofabrik_region-subregion_tier.pkl │ ├── geofabrik_subregion_names.pkl │ └── metadata.json ├── downloader │ ├── __init__.py │ ├── _downloader.py │ ├── bbbike.py │ └── geofabrik.py ├── errors.py ├── ios │ ├── __init__.py │ ├── _ios.py │ ├── bbbike.py │ ├── geofabrik.py │ └── utils.py ├── reader │ ├── __init__.py │ ├── _reader.py │ ├── bbbike.py │ ├── geofabrik.py │ ├── parser.py │ └── transformer.py └── utils.py ├── pyproject.toml ├── requirements.txt ├── setup.cfg ├── setup.py └── tests ├── __init__.py ├── data └── rutland │ ├── other_relations_1.pkl │ ├── other_relations_2.pkl │ ├── points_1.pkl │ ├── points_2.pkl │ ├── rutland-latest-free.shp.zip │ └── rutland-latest.osm.pbf ├── test_downlaoder.py ├── test_errors.py ├── test_ios.py ├── test_reader.py ├── test_updater.py └── test_utils.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Environments 2 | venv/ 3 | .idea/ 4 | */vscode/ 5 | **/__pycache__/ 6 | 7 | # Sphinx documentation 8 | */build/ 9 | **/_generated/ 10 | 11 | # Jupyter Notebooks 12 | **/.ipynb_checkpoints/ 13 | 14 | # Tests and code coverage 15 | .coverage 16 | .pytest_cache/ 17 | cov_reports/ 18 | 19 | # Distribution / packaging 20 | dist/ 21 | *.egg-info/ 22 | 23 | # Bespoke items 24 | *[!_]_.py 25 | .pypirc 26 | tutorials/* 27 | cmd.md 28 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | # .readthedocs.yaml 2 | # Read the Docs configuration file 3 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 4 | 5 | # Required 6 | version: 2 7 | 8 | # Set the version of Python and other tools 9 | build: 10 | os: ubuntu-22.04 11 | tools: 12 | python: "3.9" 13 | 14 | # Build documentation in the docs/ directory with Sphinx (this is the default documentation type) 15 | sphinx: 16 | builder: html 17 | configuration: docs/source/conf.py 18 | fail_on_warning: true 19 | 20 | # Optionally build the docs in additional formats such as PDF 21 | formats: 22 | - pdf 23 | 24 | # Optionally declare the Python requirements required to build the docs 25 | python: 26 | install: 27 | - requirements: docs/source/rtd-req.txt 28 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ### Changelog / Release notes 2 | 3 |
4 | 5 | #### **[2.2.0](https://github.com/mikeqfu/pydriosm/releases/tag/2.2.0)** 6 | 7 | (*19 June 2023*) 8 | 9 | ##### **Notable [changes](https://github.com/mikeqfu/pydriosm/compare/2.1.2...2.2.0) since [2.1.2](https://pypi.org/project/pydriosm/2.1.2/):** 10 | 11 | - [GDAL](https://pypi.org/project/GDAL/) is now an optional dependency and will only be required when dealing with PBF data. 12 | - Fixed bugs in the following classes: [BBBikeDownloader](https://github.com/mikeqfu/pydriosm/commit/59c9c8a7381c90cd3e706955a27462697bb2104f) and [PostgresOSM](https://github.com/mikeqfu/pydriosm/commit/496e0fc8cee4c5125cff2529e647394276392287). 13 | 14 | **For more information and detailed specifications, check out [PyDriosm 2.2.0 documentation](https://pydriosm.readthedocs.io/en/2.2.0/).** 15 | 16 |
17 | 18 | #### **[2.1.2](https://github.com/mikeqfu/pydriosm/releases/tag/2.1.2)** 19 | 20 | (*27 February 2023*) 21 | 22 | ##### **Notable [changes](https://github.com/mikeqfu/pydriosm/compare/2.1.1...2.1.2) since [2.1.1](https://pypi.org/project/pydriosm/2.1.1/):** 23 | 24 | - Improved the following methods: [SHPReadParse.merge_layer_shps()](https://github.com/mikeqfu/pydriosm/commit/392d759ae4504c5ebb9d8f1dab6471bf5743a7ff) and [BBBikeDownloader.get_subregion_index()](https://github.com/mikeqfu/pydriosm/commit/57b128310b89e40a12d72a0d72e34597d381ed2b). 25 | - Updated requirements on dependencies to adapt to SQLAlchemy 2.0+. 26 | 27 | **For more information and detailed specifications, check out [PyDriosm 2.1.2 documentation](https://pydriosm.readthedocs.io/en/2.1.2/).** 28 | 29 |
30 | 31 | #### **[2.1.1](https://github.com/mikeqfu/pydriosm/releases/tag/2.1.1)** 32 | 33 | (*9 January 2022*) 34 | 35 | ##### **Notable [changes](https://github.com/mikeqfu/pydriosm/compare/2.1.0...2.1.1) since [2.1.0](https://pypi.org/project/pydriosm/2.1.0/):** 36 | 37 | - Improved the following methods/modules (with bug fixes): 38 | - the methods [.write_to_shapefile()](https://github.com/mikeqfu/pydriosm/commit/57baec84a3d8366f6d8f6f324fbcbdf6e7f67fa6), [.read_layer_shps()](https://github.com/mikeqfu/pydriosm/commit/bec76cb0fc21b152849cbc8cccb2634b04dd59f8#diff-fc8bd4c3f1ee495f89956160ebf3736c1b8f8021e61f3eb14662439f8a781aacL1795-R2028), and [.merge_layer_shps()](https://github.com/mikeqfu/pydriosm/commit/bec76cb0fc21b152849cbc8cccb2634b04dd59f8#diff-fc8bd4c3f1ee495f89956160ebf3736c1b8f8021e61f3eb14662439f8a781aacR2105-R2343) of the class [SHPReadParse](https://github.com/mikeqfu/pydriosm/blob/7d2aa13c30c9df324a431567da69c7813e706c94/pydriosm/reader.py#L1030); 39 | - the modules [downloader](https://github.com/mikeqfu/pydriosm/commit/3404a8ad46b03e921110e695005bd47510c8a4f2#diff-5b569a7b9029a0d3c195aacccacf61b1f151777568a288bbd8703a88f67fc2f3) and [_updater](https://github.com/mikeqfu/pydriosm/commit/3404a8ad46b03e921110e695005bd47510c8a4f2#diff-3464d30b2db28f204143ad7f953d55614ece799bc0ae918cc0ad42c6497b39cf). 40 | 41 | **For more information and detailed specifications, check out [PyDriosm 2.1.1 documentation](https://pydriosm.readthedocs.io/en/2.1.1/).** 42 | 43 |
44 | 45 | #### **[2.1.0](https://github.com/mikeqfu/pydriosm/releases/tag/2.1.0)** 46 | 47 | (*20 November 2022*) 48 | 49 | ***Note that this release is a highly modified version and not compatible with any previous versions.*** 50 | 51 | ##### **Notable [changes](https://github.com/mikeqfu/pydriosm/compare/2.0.3...2.1.0) since [2.0.3](https://pypi.org/project/pydriosm/2.0.3/):** 52 | 53 | - Made major modifications and sweeping changes to the modules: [downloader](https://github.com/mikeqfu/pydriosm/commit/2761bc7f3cf265ca9621dc10f46ef6dcbcedf263), [reader](https://github.com/mikeqfu/pydriosm/commit/2e4befe4ea7847cba889aa4f983355db717e59e4), [ios](https://github.com/mikeqfu/pydriosm/commit/e508b11cf121d25356a02975481c4aaabf4ada56) and [utils](https://github.com/mikeqfu/pydriosm/commit/c69fe4f5a863eda4f925904afdf2daa2b6390c60). 54 | - Removed the module [settings](https://github.com/mikeqfu/pydriosm/commit/2916dc938a5890b0a19cd4431fd00f9292c7ec65). 55 | - Replaced the module [updater](https://github.com/mikeqfu/pydriosm/commit/ab6ec0ec4689bc719716ba36f9259834bb269a94) with [_updater](https://github.com/mikeqfu/pydriosm/commit/159ba27ff3410ff53ca4210409da02d03cdc2b7e). 56 | - Added a new module [errors](https://github.com/mikeqfu/pydriosm/commit/d9f60388bd085c375873bfc2f8cc395a6f111de3). 57 | 58 | **For more information and detailed specifications, check out [PyDriosm 2.1.0 documentation](https://pydriosm.readthedocs.io/en/2.1.0/).** 59 | 60 |
61 | 62 | #### **[2.0.3](https://github.com/mikeqfu/pydriosm/releases/tag/2.0.3)** 63 | 64 | (*25 April 2021*) 65 | 66 | ##### **Notable [changes](https://github.com/mikeqfu/pydriosm/compare/2.0.2...2.0.3) since [2.0.2](https://pypi.org/project/pydriosm/2.0.2/):** 67 | 68 | - Renamed the function [~~get_default_shp_crs()~~](https://github.com/mikeqfu/pydriosm/commit/5786c620fee89fa2a0db4c8329f7d9cabf7ea81d#diff-fc8bd4c3f1ee495f89956160ebf3736c1b8f8021e61f3eb14662439f8a781aacL694) to [get_epsg4326_wgs84_crs_ref()](https://github.com/mikeqfu/pydriosm/commit/5786c620fee89fa2a0db4c8329f7d9cabf7ea81d#diff-fc8bd4c3f1ee495f89956160ebf3736c1b8f8021e61f3eb14662439f8a781aacR733) in the module [reader](https://github.com/mikeqfu/pydriosm/blob/924f943de08a03595b3604f71591a0c34952a54e/pydriosm/reader.py). 69 | - Removed the function [get_osm_geom_object_dict()](https://github.com/mikeqfu/pydriosm/blob/d7cb423ae30dc3443139fc6063ea3ce24ed7afd9/pydriosm/utils.py#L145-L167) from the module [utils](https://github.com/mikeqfu/pydriosm/blob/924f943de08a03595b3604f71591a0c34952a54e/pydriosm/utils.py) 70 | - Changed the default package for reading/writing shapefiles from [GeoPandas](https://pypi.org/project/geopandas/) to [PyShp](https://pypi.org/project/pyshp/) (Note that [GeoPandas](https://pypi.org/project/geopandas/) would not be required for installing [PyDriosm 2.0.3](https://pypi.org/project/pydriosm/2.0.3/)+ but would still be reserved as an alternative option if available. 71 | - Modified the default download directories. 72 | - Improved the class [PostgresOSM](https://github.com/mikeqfu/pydriosm/commit/90587bb0ab7bd26d3597481d463b280cdaf1a728) in the module [ios](https://github.com/mikeqfu/pydriosm/blob/924f943de08a03595b3604f71591a0c34952a54e/pydriosm/ios.py) and the module [downloader](https://github.com/mikeqfu/pydriosm/commit/52f76723a84cd822fc002f89bf92a744cbd88141) with bug fixes. 73 | - Added the following new functions: 74 | - [get_epsg4326_wgs84_prj_ref()](https://github.com/mikeqfu/pydriosm/commit/5786c620fee89fa2a0db4c8329f7d9cabf7ea81d#diff-fc8bd4c3f1ee495f89956160ebf3736c1b8f8021e61f3eb14662439f8a781aacR765-R791), [specify_pyshp_fields()](https://github.com/mikeqfu/pydriosm/commit/5786c620fee89fa2a0db4c8329f7d9cabf7ea81d#diff-fc8bd4c3f1ee495f89956160ebf3736c1b8f8021e61f3eb14662439f8a781aacR794-R834) and [write_to_shapefile()](https://github.com/mikeqfu/pydriosm/commit/5786c620fee89fa2a0db4c8329f7d9cabf7ea81d#diff-fc8bd4c3f1ee495f89956160ebf3736c1b8f8021e61f3eb14662439f8a781aacR837-R917) to the module [reader](https://github.com/mikeqfu/pydriosm/blob/924f943de08a03595b3604f71591a0c34952a54e/pydriosm/reader.py); 75 | - [shp_shape_types_dict()](https://github.com/mikeqfu/pydriosm/commit/1a82b3d96383500898af13f6a49023f6c88f4c06#diff-262651b10b835e2d78c1c6d4157b36f97721b7a10a13f197715ee984266c3882R147-R172) and [shp_shape_types_geom_dict()](https://github.com/mikeqfu/pydriosm/commit/1a82b3d96383500898af13f6a49023f6c88f4c06#diff-262651b10b835e2d78c1c6d4157b36f97721b7a10a13f197715ee984266c3882R175-R192) to the module [utils](https://github.com/mikeqfu/pydriosm/blob/924f943de08a03595b3604f71591a0c34952a54e/pydriosm/utils.py). 76 | 77 | **For more information and detailed specifications, check out [PyDriosm 2.0.3 documentation](https://pydriosm.readthedocs.io/en/2.0.3/).** 78 | 79 |
80 | 81 | #### **[2.0.2](https://github.com/mikeqfu/pydriosm/releases/tag/2.0.2)** 82 | 83 | (*24 November 2020*) 84 | 85 | ##### **Notable [changes](https://github.com/mikeqfu/pydriosm/compare/2.0.1...2.0.2) since [2.0.1](https://pypi.org/project/pydriosm/2.0.1/):** 86 | 87 | - Added [a new parameter](https://github.com/mikeqfu/pydriosm/commit/3b4d8c3c58f40f4a405586594fa57524a8e825e8) `max_tmpfile_size` for setting the maximum size of in-memory temporary file while instantiating the classes [GeofabrikReader](https://github.com/mikeqfu/pydriosm/commit/3b4d8c3c58f40f4a405586594fa57524a8e825e8#diff-fc8bd4c3f1ee495f89956160ebf3736c1b8f8021e61f3eb14662439f8a781aacR1259) and [BBBikeReader](https://github.com/mikeqfu/pydriosm/commit/3b4d8c3c58f40f4a405586594fa57524a8e825e8#diff-fc8bd4c3f1ee495f89956160ebf3736c1b8f8021e61f3eb14662439f8a781aacR2123) for reading OSM data. 88 | - Added a new function [validate_shp_layer_names()](https://pydriosm.readthedocs.io/en/2.0.2/_generated/pydriosm.utils.validate_shp_layer_names.html) to the module [utils](https://pydriosm.readthedocs.io/en/2.0.2/utils.html). 89 | - Optimized import statements for all modules. 90 | 91 | **For more information and detailed specifications, check out [PyDriosm 2.0.2 documentation](https://pydriosm.readthedocs.io/en/2.0.2/).** 92 | 93 |
94 | 95 | #### **[2.0.1](https://github.com/mikeqfu/pydriosm/releases/tag/2.0.1)** 96 | 97 | (*19 November 2020*) 98 | 99 | ##### **Notable [changes](https://github.com/mikeqfu/pydriosm/compare/2.0.0...2.0.1) since [2.0.0](https://pypi.org/project/pydriosm/2.0.0/):** 100 | 101 | - Optimized import statements for the modules [downloader](https://github.com/mikeqfu/pydriosm/commit/a4d000a9f0e435e283e15c0a0db45049335e286c) and [reader](https://github.com/mikeqfu/pydriosm/commit/09ff2fc65986105566ee923b96284a78f503b3ba). 102 | 103 |
104 | 105 | #### **[2.0.0](https://github.com/mikeqfu/pydriosm/releases/tag/2.0.0)** 106 | 107 | (*19 November 2020*) 108 | 109 | This release introduces a highly modified version of the predecessors tagged "1.0.x". Note that this new version is not compatible with any previous versions. 110 | 111 | ##### **Notable [changes](https://github.com/mikeqfu/pydriosm/compare/1.0.20...2.0.0) since [1.0.20](https://pypi.org/project/pydriosm/1.0.20/):** 112 | 113 | - Featured with the following three new modules: 114 | - [**downloader**](https://github.com/mikeqfu/pydriosm/blob/941e9f5b45a0a356eba5a0281307f19807955357/pydriosm/downloader.py), modified from the former [download_GeoFabrik](https://github.com/mikeqfu/pydriosm/blob/371dbce63886cf22f8484337ed5ced826acfcf05/pydriosm/download_GeoFabrik.py) and [download_BBBike](https://github.com/mikeqfu/pydriosm/blob/371dbce63886cf22f8484337ed5ced826acfcf05/pydriosm/download_BBBike.py), for downloading data; 115 | - [**reader**](https://github.com/mikeqfu/pydriosm/blob/941e9f5b45a0a356eba5a0281307f19807955357/pydriosm/reader.py), modified from the former [read_GeoFabrik](https://github.com/mikeqfu/pydriosm/blob/371dbce63886cf22f8484337ed5ced826acfcf05/pydriosm/read_GeoFabrik.py), for reading the data; 116 | - [**ios**](https://github.com/mikeqfu/pydriosm/blob/941e9f5b45a0a356eba5a0281307f19807955357/pydriosm/ios.py), modified from the former [osm_psql](https://github.com/mikeqfu/pydriosm/blob/371dbce63886cf22f8484337ed5ced826acfcf05/pydriosm/osm_psql.py) and [dump_GeoFabrik](https://github.com/mikeqfu/pydriosm/blob/371dbce63886cf22f8484337ed5ced826acfcf05/pydriosm/dump_GeoFabrik.py), for PostgreSQL-based I/O and storage of the data. 117 | - Renamed the rest of the modules, fixed known bugs and added a number of new functions/classes. 118 | 119 |
120 | 121 | #### **[1.0.20](https://github.com/mikeqfu/pydriosm/releases/tag/1.0.20)** 122 | 123 | (*27 January 2020*) 124 | 125 | *Note that [1.0.19](https://pypi.org/project/pydriosm/1.0.19/) had been removed from [GitHub Releases](https://github.com/mikeqfu/pydriosm/releases).* 126 | 127 | ##### **Notable [changes](https://github.com/mikeqfu/pydriosm/compare/1.0.18...1.0.20) since [1.0.18](https://pypi.org/project/pydriosm/1.0.18/):** 128 | 129 | - Removed the function [split_list()](https://github.com/mikeqfu/pydriosm/commit/e3399e8ac602332aa15ddcfb23cece572906d7f4#diff-262651b10b835e2d78c1c6d4157b36f97721b7a10a13f197715ee984266c3882L111-L120) from the module [utils](https://github.com/mikeqfu/pydriosm/blob/e3399e8ac602332aa15ddcfb23cece572906d7f4/pydriosm/utils.py). 130 | - Improved the following class and functions with bug fixes: 131 | - [OSM](https://github.com/mikeqfu/pydriosm/commit/68f1edfd77ab8a9cc78cccf5197b245edf91dd19) in the module [osm_psql](https://github.com/mikeqfu/pydriosm/blob/68f1edfd77ab8a9cc78cccf5197b245edf91dd19/pydriosm/osm_psql.py); 132 | - [regulate_input_subregion_name()](https://github.com/mikeqfu/pydriosm/commit/57511fdc6948b9eb86eb07b99f4b15e1f8161dc9) in [download_GeoFabrik](https://github.com/mikeqfu/pydriosm/blob/57511fdc6948b9eb86eb07b99f4b15e1f8161dc9/pydriosm/download_GeoFabrik.py); 133 | - [read_pbf()](https://github.com/mikeqfu/pydriosm/commit/a3384eea4a628a7b2e75b19d5e9976e7172ece99#diff-c8b9e0cb8aea477d560c1f28ff9d49c58879751c45aea29eb89176cecb41ac0cL512-R602) and [read_pbf()](https://github.com/mikeqfu/pydriosm/commit/a3384eea4a628a7b2e75b19d5e9976e7172ece99#diff-c8b9e0cb8aea477d560c1f28ff9d49c58879751c45aea29eb89176cecb41ac0cL607-R695) in the module [read_GeoFabrik](https://github.com/mikeqfu/pydriosm/blob/a3384eea4a628a7b2e75b19d5e9976e7172ece99/pydriosm/read_GeoFabrik.py); 134 | - [psql_osm_pbf_data_extracts()](https://github.com/mikeqfu/pydriosm/commit/f6b0ef15bde37dd9dc65864003cb36c49b671aec) in [dump_GeoFabrik](https://github.com/mikeqfu/pydriosm/blob/f6b0ef15bde37dd9dc65864003cb36c49b671aec/pydriosm/dump_GeoFabrik.py). 135 | 136 |
137 | 138 | #### **[1.0.18](https://github.com/mikeqfu/pydriosm/releases/tag/1.0.18)** 139 | 140 | (*9 January 2020*) 141 | 142 | ##### **Notable [changes](https://github.com/mikeqfu/pydriosm/compare/1.0.17...1.0.18) since [1.0.17](https://pypi.org/project/pydriosm/1.0.17/):** 143 | 144 | - Integrated the function [collect_bbbike_subregion_download_catalogue()](https://github.com/mikeqfu/pydriosm/commit/92df65fdde05b554b732222942796eb9292e0677#diff-1adc77ee4baedd4f5bf14bb36545835023a25c9f5df660bb5fa33b1082c33688L94) into [collect_bbbike_download_catalogue()](https://github.com/mikeqfu/pydriosm/commit/92df65fdde05b554b732222942796eb9292e0677) in the module [download_BBBike](https://github.com/mikeqfu/pydriosm/blob/92df65fdde05b554b732222942796eb9292e0677/pydriosm/download_BBBike.py). 145 | - Modified the module [download_GeoFabrik](https://github.com/mikeqfu/pydriosm/commit/af47dfb667a721be97ec9ae5eac0000b4571876b#diff-a2d854a6efc7bb0057ad30f933a3cd9ac250a85d4ab74181644827157659939e), allowing it to download data of a deep or shallow set of subregions. 146 | - Improved the following functions with bug fixes: 147 | - [get_subregion_table()](https://github.com/mikeqfu/pydriosm/commit/d3b559f4b14b768eb657d471357ba621b14356a1#diff-a2d854a6efc7bb0057ad30f933a3cd9ac250a85d4ab74181644827157659939eL59-R119) in the module [download_GeoFabrik](https://github.com/mikeqfu/pydriosm/blob/d3b559f4b14b768eb657d471357ba621b14356a1/pydriosm/download_GeoFabrik.py) 148 | - [find_osm_shp_file()](https://github.com/mikeqfu/pydriosm/commit/a2627d8ec1e816f5d349dd9ff272f29a1faa7f2e#diff-c8b9e0cb8aea477d560c1f28ff9d49c58879751c45aea29eb89176cecb41ac0cL24-R58), [merge_multi_shp()](https://github.com/mikeqfu/pydriosm/commit/a2627d8ec1e816f5d349dd9ff272f29a1faa7f2e#diff-c8b9e0cb8aea477d560c1f28ff9d49c58879751c45aea29eb89176cecb41ac0cL139-R246) and [read_pbf()](https://github.com/mikeqfu/pydriosm/commit/b33b6296ede78ebff0af4753007cc3e22b691835#diff-c8b9e0cb8aea477d560c1f28ff9d49c58879751c45aea29eb89176cecb41ac0cL511-R603) in the module [read_GeoFabrik](https://github.com/mikeqfu/pydriosm/blob/b33b6296ede78ebff0af4753007cc3e22b691835/pydriosm/read_GeoFabrik.py) 149 | - Added a new module [update](https://github.com/mikeqfu/pydriosm/commit/2f96a487e2fce263772853b5a50fe7037d443697). 150 | - Added default parameters for PostgreSQL database connection. 151 | 152 |
153 | 154 | #### **[1.0.17](https://github.com/mikeqfu/pydriosm/releases/tag/1.0.17)** 155 | 156 | (*29 November 2019*) 157 | 158 | ##### **Notable [changes](https://github.com/mikeqfu/pydriosm/compare/1.0.16...1.0.17) since [1.0.16](https://pypi.org/project/pydriosm/1.0.16/):** 159 | 160 | - Improved the following functions with bug fixes: 161 | - [collect_subregion_info_catalogue()](https://github.com/mikeqfu/pydriosm/commit/8c39e6be675f163221009b4e6c66c4db904c3ccf#diff-a2d854a6efc7bb0057ad30f933a3cd9ac250a85d4ab74181644827157659939eL126-R209) and [get_default_pathname()](https://github.com/mikeqfu/pydriosm/commit/8c39e6be675f163221009b4e6c66c4db904c3ccf#diff-a2d854a6efc7bb0057ad30f933a3cd9ac250a85d4ab74181644827157659939eL455-R511) in the module [download_GeoFabrik](https://github.com/mikeqfu/pydriosm/blob/8c39e6be675f163221009b4e6c66c4db904c3ccf/pydriosm/download_GeoFabrik.py); 162 | - [merge_multi_shp()](https://github.com/mikeqfu/pydriosm/commit/94d075441dfa6ace22eec6c7c24217a2fcc2343b#diff-c8b9e0cb8aea477d560c1f28ff9d49c58879751c45aea29eb89176cecb41ac0cL140-R218), [read_shp_zip()](https://github.com/mikeqfu/pydriosm/commit/94d075441dfa6ace22eec6c7c24217a2fcc2343b#diff-c8b9e0cb8aea477d560c1f28ff9d49c58879751c45aea29eb89176cecb41ac0cL252-R359) and [read_pbf()](https://github.com/mikeqfu/pydriosm/commit/94d075441dfa6ace22eec6c7c24217a2fcc2343b#diff-c8b9e0cb8aea477d560c1f28ff9d49c58879751c45aea29eb89176cecb41ac0cL559-R661) in the module [read_GeoFabrik](https://github.com/mikeqfu/pydriosm/blob/94d075441dfa6ace22eec6c7c24217a2fcc2343b/pydriosm/read_GeoFabrik.py). 163 | 164 |
165 | 166 | #### **[1.0.16](https://github.com/mikeqfu/pydriosm/releases/tag/1.0.16)** 167 | 168 | (*6 October 2019*) 169 | 170 | ##### **Notable [changes](https://github.com/mikeqfu/pydriosm/compare/1.0.15...1.0.16) since [1.0.15](https://pypi.org/project/pydriosm/1.0.15/):** 171 | 172 | - Fixed some known bugs. 173 | 174 |
175 | 176 | #### **[1.0.15](https://github.com/mikeqfu/pydriosm/releases/tag/1.0.15)** 177 | 178 | (*29 August 2019*) 179 | 180 | *Note that [1.0.14](https://pypi.org/project/pydriosm/1.0.14/), [1.0.13](https://pypi.org/project/pydriosm/1.0.13/) and [1.0.12](https://pypi.org/project/pydriosm/1.0.12/) had been removed from [GitHub Releases](https://github.com/mikeqfu/pydriosm/releases).* 181 | 182 | ##### **Notable [changes](https://github.com/mikeqfu/pydriosm/compare/1.0.11...1.0.15) since [1.0.11](https://pypi.org/project/pydriosm/1.0.11/):** 183 | 184 | - Improved the functions: [extract_shp_zip()](https://github.com/mikeqfu/pydriosm/commit/758bcbd4dc48a03b1bb72c161ba8e87f04a80a82#diff-c8b9e0cb8aea477d560c1f28ff9d49c58879751c45aea29eb89176cecb41ac0cL74-R114), [read_shp()](https://github.com/mikeqfu/pydriosm/commit/758bcbd4dc48a03b1bb72c161ba8e87f04a80a82#diff-c8b9e0cb8aea477d560c1f28ff9d49c58879751c45aea29eb89176cecb41ac0cL168-R214) and [read_shp_zip()](https://github.com/mikeqfu/pydriosm/commit/758bcbd4dc48a03b1bb72c161ba8e87f04a80a82#diff-c8b9e0cb8aea477d560c1f28ff9d49c58879751c45aea29eb89176cecb41ac0cL193-R296) with bug fixes in the module [read_GeoFabrik](https://github.com/mikeqfu/pydriosm/blob/758bcbd4dc48a03b1bb72c161ba8e87f04a80a82/pydriosm/read_GeoFabrik.py). 185 | - Added a new method [OSM.db_exists()](https://github.com/mikeqfu/pydriosm/commit/73ff3b2bee1d85947d86bf32421e90dcabb7d47d#diff-cb2783bddce6ef6c0d7479f7e4ada08bdcec39cb0e9d0af83a4d1398b5737491R72-R76), allowing [OSM.create_db()](https://github.com/mikeqfu/pydriosm/commit/73ff3b2bee1d85947d86bf32421e90dcabb7d47d#diff-cb2783bddce6ef6c0d7479f7e4ada08bdcec39cb0e9d0af83a4d1398b5737491L72-R96) to check whether a database exists. 186 | - Updated the [LICENSE](https://github.com/mikeqfu/pydriosm/commit/90d12a5aaa36882115e89e5e9f7672b9058f7cda). 187 | 188 |
189 | 190 | #### **[1.0.11](https://github.com/mikeqfu/pydriosm/releases/tag/1.0.11)** 191 | 192 | (*4 April 2019*) 193 | 194 | *Note that [1.0.10](https://pypi.org/project/pydriosm/1.0.10/) and [1.0.9](https://pypi.org/project/pydriosm/1.0.9/) had been removed from [GitHub Releases](https://github.com/mikeqfu/pydriosm/releases).* 195 | 196 | ##### **Notable [changes](https://github.com/mikeqfu/pydriosm/compare/1.0.8...1.0.11) since [1.0.8](https://pypi.org/project/pydriosm/1.0.8/):** 197 | 198 | - Fixed [a minor bug](https://github.com/mikeqfu/pydriosm/commit/f2b22a5af3e7026c7c0810b1857550249c9fc61a) for creating a default data directory. 199 | - Improved the following functions (with bug fixes): 200 | - [get_default_pathname()](https://github.com/mikeqfu/pydriosm/commit/f2b22a5af3e7026c7c0810b1857550249c9fc61a) in the module [download_GeoFabrik](https://github.com/mikeqfu/pydriosm/blob/42c47d60c1a30c37c80b9757fd4c32e60f053bd3/pydriosm/download_GeoFabrik.py); 201 | - [parse_layer_data()](https://github.com/mikeqfu/pydriosm/commit/ec968392139282e8c66d7d0c477f9e6c5967e56c#diff-c8b9e0cb8aea477d560c1f28ff9d49c58879751c45aea29eb89176cecb41ac0cL297-R375) in the module [read_GeoFabrik](https://github.com/mikeqfu/pydriosm/blob/42c47d60c1a30c37c80b9757fd4c32e60f053bd3/pydriosm/read_GeoFabrik.py); 202 | - [dump_osm_pbf_data_by_layer()](https://github.com/mikeqfu/pydriosm/commit/ec968392139282e8c66d7d0c477f9e6c5967e56c#diff-cb2783bddce6ef6c0d7479f7e4ada08bdcec39cb0e9d0af83a4d1398b5737491L170-R179) in the module [osm_psql](https://github.com/mikeqfu/pydriosm/blob/42c47d60c1a30c37c80b9757fd4c32e60f053bd3/pydriosm/osm_psql.py); 203 | - [psql_osm_pbf_data_extracts()](https://github.com/mikeqfu/pydriosm/commit/ec968392139282e8c66d7d0c477f9e6c5967e56c#diff-06caa7c5b7806a98b9c915f4b9e44a9b7c305ead0c66e31baae94a58370c4615L68-R74) in the module [dump_GeoFabrik](https://github.com/mikeqfu/pydriosm/blob/42c47d60c1a30c37c80b9757fd4c32e60f053bd3/pydriosm/dump_GeoFabrik.py), [with a new parameter](https://github.com/mikeqfu/pydriosm/commit/9846653bb2d08580b972a0dbf10c84b1e8bd9050) `database_name` for specifying a database name when dumping data to a PostgreSQL server. 204 | - Added a function [regulate_table_name()](https://github.com/mikeqfu/pydriosm/commit/4cfdd7ebcb489b7b618f6c6163cad9354c071b77#diff-cb2783bddce6ef6c0d7479f7e4ada08bdcec39cb0e9d0af83a4d1398b5737491R17-R27), which regulates PostgreSQL table names, to the module [osm_psql](https://github.com/mikeqfu/pydriosm/blob/305be3f0996be2aa3f5003c3f96b06466d769f50/pydriosm/osm_psql.py). 205 | 206 |
207 | 208 | #### **[1.0.8](https://github.com/mikeqfu/pydriosm/releases/tag/1.0.8)** 209 | 210 | (*18 Mar 2019*) 211 | 212 | *Note that [1.0.6](https://pypi.org/project/pydriosm/1.0.6/) and [1.0.7](https://pypi.org/project/pydriosm/1.0.7/) had been removed from [GitHub Releases](https://github.com/mikeqfu/pydriosm/releases).* 213 | 214 | ##### **Notable [changes](https://github.com/mikeqfu/pydriosm/compare/1.0.5...1.0.8) since [1.0.5](https://pypi.org/project/pydriosm/1.0.5/):** 215 | 216 | - Made [some major changes](https://github.com/mikeqfu/pydriosm/commit/a415ed5d8b6394342a9ac9fb53bb041a3133fa44) (with potential bug fixes). 217 | - Fixed minor bugs in the following functions: 218 | - [parse_layer_data()](https://github.com/mikeqfu/pydriosm/commit/d266c3e49cf8a0d4e1065e60c5e3a6a657ff9332) and [read_shp_zip()](https://github.com/mikeqfu/pydriosm/commit/613f0a9fb3c70db9094590e9f614f254000369ce) in the module [read_GeoFabrik](https://github.com/mikeqfu/pydriosm/blob/305be3f0996be2aa3f5003c3f96b06466d769f50/pydriosm/read_GeoFabrik.py); 219 | - [retrieve_subregions()](https://github.com/mikeqfu/pydriosm/commit/bfbbbb4fc71108845df8fc1eae5d590a7b693d92) and [psql_subregion_osm_data_extracts()](https://github.com/mikeqfu/pydriosm/commit/f6fada22e192a56bd2d6fc250bdaedf9f6d00041) in the module [dump_GeoFabrik](https://github.com/mikeqfu/pydriosm/blob/305be3f0996be2aa3f5003c3f96b06466d769f50/pydriosm/dump_GeoFabrik.py). 220 | - Added a function [regulate_input_data_dir()](https://github.com/mikeqfu/pydriosm/commit/93792ec3493d0b13237cf24c531353f0e77d8f67) to the module [utils](https://github.com/mikeqfu/pydriosm/blob/305be3f0996be2aa3f5003c3f96b06466d769f50/pydriosm/utils.py). 221 | 222 |
223 | 224 | #### **[1.0.5](https://github.com/mikeqfu/pydriosm/releases/tag/1.0.5)** 225 | 226 | (*11 March 2019*) 227 | 228 | *Note that [1.0.4](https://pypi.org/project/pydriosm/1.0.4/), [1.0.3](https://pypi.org/project/pydriosm/1.0.3/), [1.0.2](https://pypi.org/project/pydriosm/1.0.2/) and [1.0.1](https://pypi.org/project/pydriosm/1.0.1/) had been removed from [GitHub Releases](https://github.com/mikeqfu/pydriosm/releases).* 229 | 230 | ##### **Notable [changes](https://github.com/mikeqfu/pydriosm/compare/1.0.0...1.0.5) since [1.0.0](https://pypi.org/project/pydriosm/1.0.0/):** 231 | 232 | - Integrated the function [read_parsed_osm_pbf()](https://github.com/mikeqfu/pydriosm/blob/243788f02c10fa91024b165819b52e6973fa3b26/pydriosm/read_GeoFabrik.py#L474-L515) into [read_pbf()](https://github.com/mikeqfu/pydriosm/blob/243788f02c10fa91024b165819b52e6973fa3b26/pydriosm/read_GeoFabrik.py#L319-L391) in the module [read_GeoFabrik](https://github.com/mikeqfu/pydriosm/blob/a1fb4ddce3f451e29f79fc1e06b999ab9e4eb0b2/pydriosm/read_GeoFabrik.py). 233 | - Improved the following functions 234 | - [dump_osm_pbf_data()](https://github.com/mikeqfu/pydriosm/commit/cd209d985a3270b90d22501fdc5e3a8e8b142ac4#diff-cb2783bddce6ef6c0d7479f7e4ada08bdcec39cb0e9d0af83a4d1398b5737491L173-R204) in the module [osm_psql](https://github.com/mikeqfu/pydriosm/blob/cd209d985a3270b90d22501fdc5e3a8e8b142ac4/pydriosm/osm_psql.py), with a new parameter `chunk_size` allowing users to parse/read/dump data in a chunk-wise way; 235 | - [psql_subregion_osm_data_extracts()](https://github.com/mikeqfu/pydriosm/blob/9b37bbe76223332b037f12a8fa49d1fdb24c7262/pydriosm/dump_GeoFabrik.py#L45-L152) in the module [dump_GeoFabrik](https://github.com/mikeqfu/pydriosm/blob/4558a89938fa6f28ab105a6ee5d54a95745302e2/pydriosm/dump_GeoFabrik.py); 236 | - [gdal_configurations()](https://github.com/mikeqfu/pydriosm/commit/d85af9a3a37cbb8ffb2280090844dc61bde706f2) in the module [settings](https://github.com/mikeqfu/pydriosm/blob/d85af9a3a37cbb8ffb2280090844dc61bde706f2/pydriosm/settings.py). 237 | - Added new function: 238 | - [retrieve_subregions()](https://github.com/mikeqfu/pydriosm/commit/4558a89938fa6f28ab105a6ee5d54a95745302e2#diff-06caa7c5b7806a98b9c915f4b9e44a9b7c305ead0c66e31baae94a58370c4615R19-R40), which retrieves a list of subregions of a given (sub)region name, to the module [dump_GeoFabrik](https://github.com/mikeqfu/pydriosm/blob/4558a89938fa6f28ab105a6ee5d54a95745302e2/pydriosm/dump_GeoFabrik.py); 239 | - [split_list()](https://github.com/mikeqfu/pydriosm/commit/243788f02c10fa91024b165819b52e6973fa3b26#diff-262651b10b835e2d78c1c6d4157b36f97721b7a10a13f197715ee984266c3882R242-R249) to the module [utils](https://github.com/mikeqfu/pydriosm/blob/243788f02c10fa91024b165819b52e6973fa3b26/pydriosm/utils.py). 240 | 241 |
242 | 243 | #### **[1.0.0](https://github.com/mikeqfu/pydriosm/releases/tag/1.0.0)** 244 | 245 | (*4 March 2019*) 246 | 247 | This is a release of a **brand-new** version. 248 | 249 | *Note that the initial releases (of early versions up to **~~0.2.9~~**) had been permanently deleted.* -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | # Include package data 2 | recursive-include pydriosm/data * 3 | 4 | # Exclude tests 5 | recursive-exclude tests * 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PyDriosm 2 | 3 | [![PyPI](https://img.shields.io/pypi/v/pydriosm)](https://pypi.org/project/pydriosm/) 4 | [![Python Version](https://img.shields.io/pypi/pyversions/pydriosm)](https://docs.python.org/3/) 5 | [![Documentation Status](https://readthedocs.org/projects/pydriosm/badge/?version=latest)](https://pydriosm.readthedocs.io/en/latest/?badge=latest) 6 | [![License](https://img.shields.io/pypi/l/pydriosm)](https://github.com/mikeqfu/pydriosm/blob/master/LICENSE) 7 | [![Codacy Badge](https://app.codacy.com/project/badge/Grade/b411ce89cbc445f58377a5799646d4cb)](https://app.codacy.com/gh/mikeqfu/pydriosm/dashboard?utm_source=gh&utm_medium=referral&utm_content=&utm_campaign=Badge_grade) 8 | [![DOI](https://zenodo.org/badge/92493726.svg)](https://zenodo.org/badge/latestdoi/92493726) 9 | 10 | PyDriosm is an open-source tool that provides an effortless way to download and access [OpenStreetMap](https://www.openstreetmap.org/) (OSM) data in popular file formats, such as [shapefile](https://wiki.openstreetmap.org/wiki/Shapefiles) and [protobuf binary format](https://wiki.openstreetmap.org/wiki/PBF_Format) (PBF), which are freely available from [Geofabrik](https://download.geofabrik.de/) and [BBBike](https://download.bbbike.org/). Additionally, the package offers a comprehensive solution for convenient I/O operations and efficient storage capabilities for parsed OSM data within [PostgreSQL](https://www.postgresql.org/) databases. This means that users can easily read from and write to PostgreSQL databases, enabling efficient data manipulation, querying, and other essential tasks. Whether you are a researcher, practitioner, or simply interested in working with OSM data, PyDriosm can be useful and helpful to streamline your workflow and enhance your experience. 11 | 12 | ## Installation 13 | 14 | To install the latest release of PyDriosm from [PyPI](https://pypi.org/project/pydriosm/) via [pip](https://pip.pypa.io/en/stable/cli/pip/): 15 | 16 | ```bash 17 | pip install --upgrade pydriosm 18 | ``` 19 | 20 | Please also refer to [Installation](https://pydriosm.readthedocs.io/en/latest/installation.html) for more information. 21 | 22 | ## Quick start 23 | 24 | For a concise guide with practical examples, please check out the [quick-start tutorial](https://pydriosm.readthedocs.io/en/latest/quick-start.html). This tutorial showcases how to utilise PyDriosm for various tasks, such as downloading, parsing, and performing storage I/O operations on OSM data using a PostgreSQL database. 25 | 26 | ## Documentation 27 | 28 | The complete PyDriosm documentation: [[HTML](https://pydriosm.readthedocs.io/en/latest/)\] \[[PDF](https://pydriosm.readthedocs.io/_/downloads/en/latest/pdf/)] 29 | 30 | It is hosted on [ReadTheDocs](https://readthedocs.org/projects/pydriosm/) and provides a wealth of detailed examples. 31 | 32 | ## License 33 | 34 | - PyDriosm is licensed under [GNU General Public License v3.0](https://github.com/mikeqfu/pydriosm/blob/master/LICENSE) or later (GPLv3+). 35 | - The free [OpenStreetMap](https://www.openstreetmap.org/) data, which is used for the development of PyDriosm, is licensed under the [Open Data Commons Open Database License](https://opendatacommons.org/licenses/odbl/) (ODbL) by the [OpenStreetMap Foundation](https://osmfoundation.org/) (OSMF). 36 | 37 | ## Acknowledgement 38 | 39 | The development of PyDriosm, including the example code that demonstrates how to use the package, heavily relies on freely available [OpenStreetMap](https://www.openstreetmap.org/) data. The author would like to express sincere gratitude to all the [OpenStreetMap contributors](https://wiki.openstreetmap.org/wiki/Contributors) for their invaluable contributions in making this data accessible to the community. 40 | 41 | ## Cite as 42 | 43 | Fu, Q. (2020). PyDriosm: an open-source tool for downloading, reading and PostgreSQL-based I/O of OpenStreetMap data. Zenodo. [doi:10.5281/zenodo.4281194](https://doi.org/10.5281/zenodo.4281194) 44 | 45 | ```bibtex 46 | @software{qian_fu_pydriosm_4281194, 47 | author = {Qian Fu}, 48 | title = {{PyDriosm: an open-source tool for downloading, reading 49 | and PostgreSQL-based I/O of OpenStreetMap data}}, 50 | year = 2020, 51 | publisher = {Zenodo}, 52 | doi = {10.5281/zenodo.4718623}, 53 | url = {https://doi.org/10.5281/zenodo.4281194} 54 | } 55 | ``` 56 | 57 | (Please also refer to the export options from [Zenodo](https://zenodo.org/search?page=1&size=20&q=conceptrecid:4281194&all_versions&sort=-version) to reference the specific version of PyDriosm as appropriate.) 58 | 59 | ## Contributors 60 | 61 | 62 | 63 | 64 | 65 | 72 | 73 |
66 | Qian Fu
Qian Fu

67 | 🌱 68 | 💻 69 | 🧪 70 | 📖 71 |
-------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = source 9 | BUILDDIR = build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=source 11 | set BUILDDIR=build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.http://sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /docs/source/_images/GeometryCollection.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/docs/source/_images/GeometryCollection.png -------------------------------------------------------------------------------- /docs/source/_images/LineString.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/docs/source/_images/LineString.png -------------------------------------------------------------------------------- /docs/source/_images/MultiLineString.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/docs/source/_images/MultiLineString.png -------------------------------------------------------------------------------- /docs/source/_images/MultiPolygon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/docs/source/_images/MultiPolygon.png -------------------------------------------------------------------------------- /docs/source/_images/Point.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/docs/source/_images/Point.png -------------------------------------------------------------------------------- /docs/source/_images/pbf_db_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/docs/source/_images/pbf_db_example.png -------------------------------------------------------------------------------- /docs/source/_images/pbf_schemas_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/docs/source/_images/pbf_schemas_example.png -------------------------------------------------------------------------------- /docs/source/_images/pbf_schemas_example_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/docs/source/_images/pbf_schemas_example_2.png -------------------------------------------------------------------------------- /docs/source/_images/pbf_table_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/docs/source/_images/pbf_table_example.png -------------------------------------------------------------------------------- /docs/source/_static/copy_button.css: -------------------------------------------------------------------------------- 1 | /* Copy buttons */ 2 | button.copybtn { 3 | position: absolute; 4 | display: flex; 5 | top: auto; 6 | bottom: .5em; /* 1.5em; */ 7 | right: .2em; /* 2.0em; */ 8 | width: 1.7em; 9 | height: 1.7em; 10 | opacity: 0; 11 | transition: opacity 0.3s, border .3s, background-color .3s; 12 | user-select: none; 13 | padding: 0; 14 | border: none; /* #e1e1e1 1px solid; */ 15 | outline: none; 16 | border-radius: 0.4em; 17 | background-color: rgb(245, 245, 245); 18 | } 19 | 20 | button.copybtn.success { 21 | border-color: #22863a; 22 | } 23 | 24 | button.copybtn img { 25 | width: 100%; 26 | padding: .2em; 27 | } 28 | 29 | div.highlight { 30 | position: relative; 31 | } 32 | 33 | .highlight:hover button.copybtn { 34 | opacity: 1; 35 | } 36 | 37 | .highlight button.copybtn:hover { 38 | background-color: rgb(235, 235, 235); 39 | } 40 | 41 | .highlight button.copybtn:active { 42 | background-color: rgb(187, 187, 187); 43 | } 44 | 45 | /** 46 | * A minimal CSS-only tooltip copied from: 47 | * https://codepen.io/mildrenben/pen/rVBrpK 48 | * 49 | * To use, write HTML like the following: 50 | * 51 | *

Short

52 | */ 53 | .o-tooltip--left { 54 | position: relative; 55 | } 56 | 57 | .o-tooltip--left:after { 58 | opacity: 0; 59 | visibility: hidden; 60 | position: absolute; 61 | content: attr(data-tooltip); 62 | padding: .2em; 63 | font-size: .8em; 64 | left: -.2em; 65 | background: grey; 66 | color: white; 67 | white-space: nowrap; 68 | z-index: 2; 69 | border-radius: 2px; 70 | transform: translateX(-102%) translateY(0); 71 | transition: opacity 0.2s cubic-bezier(0.64, 0.09, 0.08, 1), transform 0.2s cubic-bezier(0.64, 0.09, 0.08, 1); 72 | } 73 | 74 | .o-tooltip--left:hover:after { 75 | display: block; 76 | opacity: 1; 77 | visibility: visible; 78 | transform: translateX(-100%) translateY(0); 79 | transition: opacity 0.2s cubic-bezier(0.64, 0.09, 0.08, 1), transform 0.2s cubic-bezier(0.64, 0.09, 0.08, 1); 80 | transition-delay: .5s; 81 | } 82 | -------------------------------------------------------------------------------- /docs/source/_static/prompt_button.js: -------------------------------------------------------------------------------- 1 | $(document).ready(function() { 2 | /* Add a [>>>] button on the top-right corner of code samples to hide the >>> and ... prompts 3 | * and the output, thus making the code copyable. 4 | * Resource: https://docs.python.org/3/_static/copybutton.js */ 5 | let div = $('.highlight-python .highlight,' + 6 | '.highlight-python3 .highlight,' + 7 | '.highlight-pycon .highlight,' + 8 | '.highlight-default .highlight'); 9 | let pre = div.find('pre'); 10 | 11 | // Get the styles from the current theme 12 | pre.parent().parent().css('position', 'relative'); 13 | let hide_text = 'Hide the prompts and output'; 14 | let show_text = 'Show the prompts and output'; 15 | let border_width = pre.css('border-top-width'); 16 | let border_style = pre.css('border-top-style'); 17 | let border_color = pre.css('border-top-color'); 18 | let button_styles = { 19 | 'cursor':'pointer', 20 | 'position': 'absolute', 21 | 'top': '0', 22 | 'right': '0', 23 | 'border-color': border_color, 24 | 'border-style': border_style, 25 | 'border-width': border_width, 26 | 'color': border_color, 27 | 'text-size': '75%', 28 | 'font-family': 'monospace', 29 | 'padding-left': '0.2em', 30 | 'padding-right': '0.2em', 31 | 'border-radius': '0 3px 0 0' 32 | } 33 | 34 | // Create and add the button to all the code blocks that contain >>> 35 | div.each(function() { 36 | let jthis = $(this); 37 | if (jthis.find('.gp').length > 0) { 38 | let button = $('>>>'); 39 | button.css(button_styles) 40 | button.attr('title', hide_text); 41 | button.data('hidden', 'false'); 42 | jthis.prepend(button); 43 | } 44 | /* tracebacks (.gt) contain bare text elements that need to be wrapped 45 | * in a span to work with .nextUntil() (see later) */ 46 | jthis.find('pre:has(.gt)').contents().filter(function() { 47 | return ((this.nodeType === 3) && (this.data.trim().length > 0)); 48 | }).wrap(''); 49 | }); 50 | 51 | // Define the behavior of the button when it's clicked 52 | $('.promptbutton').click(function(e){ 53 | e.preventDefault(); 54 | let button = $(this); 55 | if (button.data('hidden') === 'false') { 56 | // hide the code output 57 | button.parent().find('.go, .gp, .gt').hide(); 58 | button.next('pre').find('.gt').nextUntil('.gp, .go').css('visibility', 'hidden'); 59 | button.css('text-decoration', 'line-through'); 60 | button.attr('title', show_text); 61 | button.data('hidden', 'true'); 62 | } else { 63 | // show the code output 64 | button.parent().find('.go, .gp, .gt').show(); 65 | button.next('pre').find('.gt').nextUntil('.gp, .go').css('visibility', 'visible'); 66 | button.css('text-decoration', 'none'); 67 | button.attr('title', hide_text); 68 | button.data('hidden', 'false'); 69 | } 70 | }); 71 | }); 72 | -------------------------------------------------------------------------------- /docs/source/_static/rtd_overrides.css: -------------------------------------------------------------------------------- 1 | @media screen and (min-width: 767px) { 2 | 3 | .wy-table-responsive table td { 4 | /* !important prevents the common CSS stylesheets from overriding 5 | this as on RTD they are loaded after this stylesheet */ 6 | white-space: normal !important; 7 | } 8 | 9 | .wy-table-responsive { 10 | overflow: visible !important; 11 | } 12 | 13 | .wy-nav-content { 14 | max-width: 72% !important; 15 | } 16 | 17 | code.literal { 18 | color: #404040 !important; 19 | background-color: #fbfbfb !important; 20 | } 21 | } -------------------------------------------------------------------------------- /docs/source/_templates/base.rst: -------------------------------------------------------------------------------- 1 | {{ objname | escape | underline}} 2 | 3 | .. currentmodule:: {{ module }} 4 | 5 | .. auto{{ objtype }}:: {{ objname }} 6 | 7 | -------------------------------------------------------------------------------- /docs/source/_templates/breadcrumbs.html: -------------------------------------------------------------------------------- 1 | {%- extends "sphinx_rtd_theme/breadcrumbs.html" %} 2 | 3 | {% block breadcrumbs_aside %} 4 | {% endblock %} -------------------------------------------------------------------------------- /docs/source/_templates/class.rst: -------------------------------------------------------------------------------- 1 | {{ objname | escape | underline }} 2 | 3 | .. currentmodule:: {{ module }}.{{ objname }} 4 | 5 | .. autoclass:: {{ module }}.{{ objname }} 6 | {% block attributes %} 7 | {% if attributes %} 8 | .. rubric:: {{ _('Attributes') }} 9 | .. autosummary:: 10 | :template: base.rst 11 | :toctree: 12 | {% for item in attributes %} 13 | {%- if (not item.startswith('_')) and (item not in inherited_members) %} 14 | {{ item }} 15 | {%- endif -%} 16 | {%- endfor %} 17 | {% endif %} 18 | {% endblock %} 19 | {% block methods %} 20 | {% if methods %} 21 | .. rubric:: {{ _('Methods') }} 22 | .. autosummary:: 23 | :template: base.rst 24 | :toctree: 25 | {% for item in methods %} 26 | {%- if (not item.startswith('_') or item in ['__call__']) and (item not in inherited_members) %} 27 | {{ item }} 28 | {%- endif -%} 29 | {%- endfor %} 30 | {% endif %} 31 | {% endblock %} 32 | 33 | -------------------------------------------------------------------------------- /docs/source/_templates/class2.rst: -------------------------------------------------------------------------------- 1 | {{ objname | escape | underline }} 2 | 3 | .. currentmodule:: {{ module }}.{{ objname }} 4 | 5 | .. autoclass:: {{ module }}.{{ objname }} 6 | {% block attributes %} 7 | {% if attributes %} 8 | .. autosummary:: 9 | :template: base.rst 10 | :toctree: 11 | {% for item in attributes %} 12 | {%- if (not item.startswith('_')) and (item not in inherited_members) %} 13 | {{ item }} 14 | {%- endif -%} 15 | {%- endfor %} 16 | {% endif %} 17 | {% endblock %} 18 | {% block methods %} 19 | {% if methods %} 20 | .. autosummary:: 21 | :template: base.rst 22 | :toctree: 23 | {% for item in methods %} 24 | {%- if (not item.startswith('_') or item in ['__call__']) and (item not in inherited_members) %} 25 | {{ item }} 26 | {%- endif -%} 27 | {%- endfor %} 28 | {% endif %} 29 | {% endblock %} 30 | 31 | -------------------------------------------------------------------------------- /docs/source/_templates/function.rst: -------------------------------------------------------------------------------- 1 | {{ objname | escape | underline }} 2 | 3 | .. currentmodule:: {{module}} 4 | 5 | .. autofunction:: {{objname}} 6 | 7 | -------------------------------------------------------------------------------- /docs/source/_templates/latex.tex_t: -------------------------------------------------------------------------------- 1 | %% Generated by Sphinx. 2 | \def\sphinxdocclass{<%= docclass %>} 3 | <% if latex_engine == 'lualatex' -%> 4 | \IfFileExists{luatex85.sty} 5 | {\RequirePackage{luatex85}} 6 | {\ifdefined\luatexversion\ifnum\luatexversion>84\relax 7 | \PackageError{sphinx} 8 | {** With this LuaTeX (\the\luatexversion),Sphinx requires luatex85.sty **} 9 | {** Add the LaTeX package luatex85 to your TeX installation, and try again **} 10 | \endinput\fi\fi} 11 | <% endif -%> 12 | \documentclass[<%= papersize %>,<%= pointsize %><%= classoptions %>]{<%= wrapperclass %>} 13 | \usepackage[table]{xcolor} 14 | \definecolor{anti-flashwhite}{rgb}{0.95, 0.95, 0.96} 15 | \ifdefined\pdfpxdimen 16 | \let\sphinxpxdimen\pdfpxdimen\else\newdimen\sphinxpxdimen 17 | \fi \sphinxpxdimen=<%= pxunit %>\relax 18 | <% if use_xindy -%> 19 | %% turn off hyperref patch of \index as sphinx.xdy xindy module takes care of 20 | %% suitable \hyperpage mark-up, working around hyperref-xindy incompatibility 21 | \PassOptionsToPackage{hyperindex=false}{hyperref} 22 | <% endif -%> 23 | <%= passoptionstopackages %> 24 | \PassOptionsToPackage{warn}{textcomp} 25 | <%= inputenc %> 26 | <%= utf8extra %> 27 | <%= cmappkg %> 28 | <%= fontenc %> 29 | <%= amsmath %> 30 | <%= multilingual %> 31 | <%= substitutefont %> 32 | <%= textcyrillic %> 33 | <%= fontpkg %> 34 | <%= textgreek %> 35 | <%= fncychap %> 36 | \usepackage<%= sphinxpkgoptions %>{sphinx} 37 | <%= sphinxsetup %> 38 | <%= fvset %> 39 | <%= geometry %> 40 | <%= extrapackages %> 41 | 42 | <%- for name, option in packages %> 43 | <%- if option %> 44 | \usepackage[<%= option %>]{<%= name %>} 45 | <%- else %> 46 | \usepackage{<%= name %>} 47 | <%- endif %> 48 | <%- endfor %> 49 | 50 | <%= hyperref %> 51 | <%- for name, option in packages_after_hyperref %> 52 | <%- if option %> 53 | \usepackage[<%= option %>]{<%= name %>} 54 | <%- else %> 55 | \usepackage{<%= name %>} 56 | <%- endif %> 57 | <%- endfor %> 58 | 59 | <%= contentsname %> 60 | \usepackage{sphinxmessages} 61 | <%= tocdepth %> 62 | <%= secnumdepth %> 63 | <%= preamble %> 64 | 65 | \title{<%= title %>} 66 | \date{<%= date %>} 67 | \release{<%= release %>} 68 | \author{<%= author %>} 69 | <%- if logofilename %> 70 | \newcommand{\sphinxlogo}{\sphinxincludegraphics{<%= logofilename %>}\par} 71 | <%- else %> 72 | \newcommand{\sphinxlogo}{\vbox{}} 73 | <%- endif %> 74 | <%- if releasename or release %> 75 | \renewcommand{\releasename}{<%= releasename or _('Release') | e %>} 76 | <%- else %> 77 | \renewcommand{\releasename}{} 78 | <%- endif %> 79 | <%= makeindex %> 80 | \begin{document} 81 | <%= shorthandoff %> 82 | \pagestyle{empty} 83 | <%= maketitle %> 84 | \pagestyle{plain} 85 | <%= tableofcontents %> 86 | \pagestyle{normal} 87 | <%= body %> 88 | <%= atendofbody %> 89 | <%= indices %> 90 | \renewcommand{\indexname}{<%= _('Index') | e %>} 91 | <%= printindex %> 92 | \end{document} 93 | -------------------------------------------------------------------------------- /docs/source/_templates/longtable.tex_t: -------------------------------------------------------------------------------- 1 | \begin{savenotes}\sphinxatlongtablestart\begin{longtable} 2 | <%- if table.align in ('center', 'default') -%> 3 | [c] 4 | <%- elif table.align == 'left' -%> 5 | [l] 6 | <%- elif table.align == 'right' -%> 7 | [r] 8 | <%- endif -%> 9 | <%= table.get_colspec() %> % {\X{6}{9}\X{3}{9}} 10 | <%- if table.caption -%> 11 | \sphinxthelongtablecaptionisattop 12 | \caption*{<%= ''.join(table.caption) %>\strut}<%= labels %>\\*[\sphinxlongtablecapskipadjust] 13 | \hline 14 | <% elif labels -%> 15 | \hline\noalign{\phantomsection<%= labels %>}% 16 | <% else -%> 17 | \hline 18 | <% endif -%> 19 | <%= ''.join(table.header) %> 20 | \endfirsthead 21 | 22 | \multicolumn{<%= table.colcount %>}{l}% 23 | {\makebox[0pt][l]{\sphinxtablecontinued{<%= _('(continued from previous page)') %>}}}\\ 24 | \hline 25 | \endhead 26 | 27 | \hline 28 | \multicolumn{<%= table.colcount %>}{r}{\makebox[0pt][r]{\sphinxtablecontinued{<%= _('(continues on next page)') %>}}}\\ 29 | \endfoot 30 | 31 | \endlastfoot 32 | <%= ''.join(table.body) %> 33 | \end{longtable}\sphinxatlongtableend\end{savenotes} 34 | -------------------------------------------------------------------------------- /docs/source/_templates/module.rst: -------------------------------------------------------------------------------- 1 | {{ module | escape | underline }} 2 | 3 | .. currentmodule:: {{ fullname }} 4 | 5 | .. automodule:: {{ fullname }} 6 | 7 | {% if functions %} 8 | .. rubric:: Functions 9 | .. autosummary:: 10 | :template: function.rst 11 | :toctree: . 12 | {% for function in functions %} 13 | {{ function }} 14 | {% endfor %} 15 | {% endif %} 16 | {% if classes %} 17 | .. rubric:: Classes 18 | .. autosummary:: 19 | :template: class.rst 20 | :toctree: . 21 | {% for class in classes %} 22 | {{ class }} 23 | {% endfor %} 24 | {% endif %} 25 | 26 | -------------------------------------------------------------------------------- /docs/source/acknowledgement.rst: -------------------------------------------------------------------------------- 1 | =============== 2 | Acknowledgement 3 | =============== 4 | 5 | The development of `pydriosm `_, including the example code that demonstrates how to use the package, heavily relies on freely available `OpenStreetMap `_ data. The author would like to express sincere gratitude to all the `OpenStreetMap contributors `_ for their invaluable contributions in making this data accessible to the community. 6 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | """ 2 | Configuration file for the Sphinx documentation builder. 3 | """ 4 | 5 | # == Path setup ==================================================================================== 6 | import os 7 | import sys 8 | 9 | # If the directory is relative to the documentation root, use os.path.abspath to make it absolute: 10 | sys.path.insert(0, os.path.abspath('../..')) 11 | sys.path.insert(0, os.path.abspath('../../pydriosm')) 12 | sys.path.insert(0, os.path.abspath('../../pydriosm/data')) 13 | 14 | # == Project information =========================================================================== 15 | from pydriosm import __affiliation__, __author__, __copyright__, __description__, __pkgname__, \ 16 | __project__, __version__, __first_release_date__ 17 | 18 | # General information about the project: 19 | project = __project__ 20 | copyright = __copyright__ 21 | 22 | # The version info for the project: 23 | version = __version__ # The short X.Y.Z version 24 | release = version # The full version, including alpha/beta/rc tags 25 | 26 | # == General configuration ========================================================================= 27 | extensions = [ # Sphinx extension module names, which can be named 'sphinx.ext.*' or custom ones: 28 | 'sphinx.ext.autodoc', 29 | 'sphinx.ext.autosummary', 30 | 'sphinx.ext.autosectionlabel', 31 | 'sphinx.ext.extlinks', 32 | 'sphinx.ext.intersphinx', 33 | 'sphinx.ext.inheritance_diagram', 34 | 'sphinx.ext.githubpages', 35 | 'sphinx.ext.todo', 36 | 'sphinx_rtd_theme', 37 | 'sphinx.ext.linkcode', 38 | 'sphinx.ext.doctest', 39 | 'sphinx_copybutton', 40 | ] 41 | 42 | 43 | def linkcode_resolve(domain, info): 44 | """ 45 | Determine the URL corresponding to Python object. 46 | 47 | (Adapted from https://github.com/pandas-dev/pandas/blob/main/doc/source/conf.py) 48 | """ 49 | 50 | import inspect 51 | import warnings 52 | 53 | import pydriosm 54 | 55 | if domain != 'py' or not info['module']: 56 | return None 57 | 58 | module_name, full_name = info['module'], info['fullname'] 59 | 60 | sub_module_name = sys.modules.get(module_name) 61 | if sub_module_name is None: 62 | return None 63 | 64 | obj = sub_module_name 65 | for part in full_name.split('.'): 66 | try: 67 | with warnings.catch_warnings(): 68 | # Accessing deprecated objects will generate noisy warnings 69 | warnings.simplefilter('ignore', FutureWarning) 70 | obj = getattr(obj, part) 71 | except AttributeError: 72 | return None 73 | 74 | try: 75 | fn = inspect.getsourcefile(inspect.unwrap(obj)) 76 | except TypeError: 77 | try: # property 78 | fn = inspect.getsourcefile(inspect.unwrap(obj.fget)) 79 | except (AttributeError, TypeError): 80 | fn = None 81 | if not fn: 82 | return None 83 | 84 | source = [0] 85 | try: 86 | source, line_no = inspect.getsourcelines(obj) 87 | except TypeError: 88 | try: # property 89 | source, line_no = inspect.getsourcelines(obj.fget) 90 | except (AttributeError, TypeError): 91 | line_no = None 92 | except OSError: 93 | line_no = None 94 | 95 | if line_no: 96 | line_spec = f"#L{line_no}-L{line_no + len(source) - 1}" 97 | else: 98 | line_spec = "" 99 | 100 | # fn = os.path.relpath(fn, start=os.path.abspath("..")) 101 | fn = os.path.relpath(fn, start=os.path.dirname(pydriosm.__file__)) 102 | 103 | # f"https://github.com/mikeqfu/pydriosm/blob/{pydriosm.__version__}/pydriosm/{fn}{line_spec}" 104 | url = f"https://github.com/mikeqfu/pydriosm/blob/master/pydriosm/{fn}{line_spec}" 105 | 106 | return url 107 | 108 | 109 | # Enable to reference numbered figures: 110 | numfig = True 111 | numfig_secnum_depth = 0 112 | numfig_format = {'figure': 'Fig. %s', 'table': 'Table %s', 'code-block': 'Code Block %s'} 113 | numfig_format_caption = {'figure': 'Fig. %s: '} 114 | 115 | # The language for content autogenerated by Sphinx: 116 | language = 'en' 117 | 118 | # Add any paths that contain templates here, relative to this directory: 119 | templates_path = ['_templates'] 120 | 121 | # Patterns (relative to source directory) that match files & directories to ignore: 122 | exclude_patterns = ['_build', '../_build', '../build'] 123 | 124 | # Whether to scan all found documents for autosummary directives and generate stub pages for each: 125 | autosummary_generate = True 126 | 127 | # The suffix(es) of source filenames (For multiple suffix, a list of string: 128 | source_suffix = '.rst' # e.g. source_suffix = ['.rst', '.md']) 129 | 130 | # The master toctree document: 131 | master_doc = 'index' 132 | 133 | # Content inserted into the main body of an `autoclass` directive: 134 | autoclass_content = 'both' # ['class', 'init'] 135 | 136 | # Automatically documented members are sorted by source order ('bysource'): 137 | autodoc_member_order = 'bysource' 138 | 139 | # == Options for HTML and HTMLHelp output ========================================================== 140 | html_theme = 'sphinx_rtd_theme' # The theme to use for HTML and HTML Help pages 141 | 142 | html_theme_options = { 143 | 'collapse_navigation': False, 144 | 'navigation_depth': 3, 145 | } 146 | 147 | # Hide/show source link: 148 | html_copy_source = False 149 | html_show_sourcelink = False 150 | 151 | # The name of the Pygments (syntax highlighting) style to use: 152 | pygments_style = 'sphinx' # or 'default' 153 | 154 | # Paths containing custom static files (e.g. style sheets), relative to this directory: 155 | html_static_path = ['_static'] 156 | 157 | # Add custom CSS: 158 | html_css_files = ['rtd_overrides.css', 'copy_button.css'] 159 | 160 | # Add custom JavaScript: 161 | html_js_files = ['prompt_button.js'] 162 | 163 | # Output file base name for HTML help builder: 164 | htmlhelp_basename = __project__ + 'doc' # Defaults to 'pydoc' 165 | 166 | # == Options for LaTeX output ====================================================================== 167 | from pygments.formatters.latex import LatexFormatter 168 | from sphinx.highlighting import PygmentsBridge 169 | 170 | 171 | class CustomLatexFormatter(LatexFormatter): 172 | def __init__(self, **options): 173 | super(CustomLatexFormatter, self).__init__(**options) 174 | self.verboptions = r"formatcom=\footnotesize" 175 | 176 | 177 | PygmentsBridge.latex_formatter = CustomLatexFormatter 178 | 179 | # The LaTeX engine to build the docs 180 | latex_engine = 'pdflatex' 181 | 182 | # Grouping the document tree into LaTeX files 183 | latex_documents = [ 184 | ('index', # source start file 185 | __pkgname__ + '.tex', # target name 186 | project + ' Documentation', # title 187 | __author__, # author 188 | 'manual', # document class ['howto', 'manual', or own class] 189 | 1 # toctree only 190 | ), 191 | ] 192 | 193 | affil_dept, affil_sch, affil_univ = __affiliation__.split(', ') 194 | 195 | # Custom title page 196 | latex_maketitle = r''' 197 | \newgeometry{top=1.1in,bottom=1.1in,right=1.0in,left=1.0in} 198 | \pagenumbering{roman} 199 | \makeatletter 200 | \hypertarget{titlepage}{} 201 | \begin{titlepage} 202 | \flushright 203 | 204 | \vspace*{22mm} 205 | \textbf{\Huge {{%s}}} 206 | 207 | \vspace{5mm} 208 | \textit{\Large {{%s}}} \par 209 | \vspace{5mm} 210 | \LARGE \textbf{\textit{{Release %s}}} \par 211 | 212 | \vspace{45mm} 213 | \LARGE \textbf{{%s}} \par 214 | \Large \textit{{%s}} \par 215 | \Large \textit{{%s}} \par 216 | \Large \textit{{%s}} \par 217 | 218 | \vspace{50mm} 219 | \Large {{First release:}} \large \textbf{{%s}} \par 220 | \Large {{Last updated:}} \large \textbf{{\MonthYearFormat\today}} \par 221 | 222 | \vspace{15mm} 223 | \large \textcopyright \space Copyright %s \par 224 | 225 | \end{titlepage} 226 | \restoregeometry 227 | \bookmark[dest=titlepage]{Title} 228 | \makeatother 229 | 230 | \clearpage 231 | \pagenumbering{roman} 232 | 233 | \cleardoublepage 234 | \makeatletter 235 | \hypertarget{tocpage}{} 236 | \tableofcontents 237 | \bookmark[dest=tocpage]{Table of Contents} 238 | \makeatother 239 | 240 | \cleardoublepage 241 | \makeatletter 242 | \hypertarget{lofpage}{} 243 | \listoffigures 244 | \bookmark[dest=lofpage]{List of Figures} 245 | \makeatother 246 | 247 | \clearpage 248 | \pagenumbering{arabic} 249 | ''' % (project, 250 | __description__.rstrip('.'), 251 | release, 252 | __author__, 253 | affil_dept, 254 | affil_sch, 255 | affil_univ, 256 | __first_release_date__, 257 | __copyright__) 258 | 259 | latex_preamble = r''' 260 | \setlength{\headheight}{14pt} 261 | \DeclareUnicodeCharacter{229E}{\ensuremath{\boxplus}} 262 | \setcounter{tocdepth}{2} 263 | \setcounter{secnumdepth}{2} 264 | \usepackage{float,textcomp,textgreek,graphicx,blindtext,color,svg,booktabs,newunicodechar} 265 | \usepackage{datetime} 266 | \newdateformat{MonthYearFormat}{% 267 | \monthname[\THEMONTH] \THEYEAR} 268 | \usepackage[none]{hyphenat} 269 | \usepackage[document]{ragged2e} 270 | \usepackage[utf8]{inputenc} 271 | \usepackage[sc,osf]{mathpazo} 272 | \linespread{1.05} 273 | \renewcommand{\sfdefault}{pplj} 274 | \newunicodechar{≤}{\ensuremath{\leq}} 275 | \newunicodechar{≈}{\ensuremath{\approx}} 276 | \newunicodechar{≥}{\ensuremath{\geq}} 277 | \IfFileExists{zlmtt.sty} 278 | {\usepackage[light,scaled=1.05]{zlmtt}} 279 | {\renewcommand{\ttdefault}{lmtt}} 280 | \let\oldlongtable\longtable 281 | \let\endoldlongtable\endlongtable 282 | \renewenvironment{longtable} 283 | {\rowcolors{1}{anti-flashwhite}{white}\oldlongtable} 284 | {\endoldlongtable} 285 | \usepackage[open]{bookmark} 286 | \bookmarksetup{numbered} 287 | 288 | \addto\captionsenglish{\renewcommand{\contentsname}{Table of Contents}} 289 | ''' 290 | 291 | # LaTeX customization: 292 | latex_elements = { 293 | 'papersize': 'a4paper', # The paper size ('letterpaper' or 'a4paper') 294 | 'pointsize': '11pt', # The font size ('10pt', '11pt' or '12pt') 295 | 'pxunit': '0.25bp', 296 | 297 | 'fontpkg': '\\usepackage{amsmath,amsfonts,amssymb,amsthm}', 298 | 'fncychap': '\\usepackage{fncychap}', # '\\usepackage[Bjarne]{fncychap}' 299 | 300 | 'preamble': latex_preamble, 301 | 302 | 'figure_align': 'H', # Latex figure (float) alignment; optional: 'htbp' 303 | 'extraclassoptions': 'openany,oneside', 304 | 305 | 'maketitle': latex_maketitle, 306 | 307 | 'releasename': ' ', 308 | 'tableofcontents': ' ', 309 | 310 | # 'makeindex': ' ', 311 | 'printindex': r''' 312 | \IfFileExists{\jobname.ind} 313 | {\footnotesize\raggedright\printindex} 314 | {\begin{sphinxtheindex}\end{sphinxtheindex}} 315 | ''', 316 | 317 | 'fvset': r'\fvset{fontsize=auto}', 318 | 319 | 'sphinxsetup': r''' 320 | %verbatimwithframe=false, 321 | %verbatimwrapslines=false, 322 | %verbatimhintsturnover=false, 323 | VerbatimColor={HTML}{F5F5F5}, 324 | VerbatimBorderColor={HTML}{E0E0E0}, 325 | noteBorderColor={HTML}{E0E0E0}, 326 | noteborder=1.5pt, 327 | warningBorderColor={HTML}{E0E0E0}, 328 | warningborder=1.5pt, 329 | warningBgColor={HTML}{FBFBFB}, 330 | hmargin={0.7in,0.7in}, vmargin={0.9in,0.9in}, 331 | ''', 332 | 333 | 'passoptionstopackages': r'\PassOptionsToPackage{svgnames}{xcolor}', 334 | } 335 | 336 | # == Options for manual page output ================================================================ 337 | man_pages = [ # How to group the document tree into manual pages 338 | ('index', # startdocname 339 | __pkgname__, # name 340 | project + ' Documentation', # description 341 | [__author__], # authors 342 | 1 # section 343 | ) 344 | ] 345 | 346 | # == Options for Texinfo output ==================================================================== 347 | texinfo_documents = [ # Grouping the document tree into Texinfo files 348 | (master_doc, # source start file 349 | __pkgname__, # target name 350 | project + ' Documentation', # title 351 | __author__, # author 352 | project, # dir menu entry 353 | __description__, # description 354 | 'Data manipulation tools', # category 355 | 1 # toctree only 356 | ), 357 | ] 358 | -------------------------------------------------------------------------------- /docs/source/contributors.rst: -------------------------------------------------------------------------------- 1 | ============ 2 | Contributors 3 | ============ 4 | 5 | .. only:: html 6 | 7 | .. raw:: html 8 | 9 | 10 | 11 | 12 | 19 | 20 |
13 | Qian Fu
Qian Fu

14 | 🌱 15 | 💻 16 | 🧪 17 | 📖 18 |
21 |

22 | 23 | .. only:: latex 24 | 25 | - `Qian Fu `_ 26 | -------------------------------------------------------------------------------- /docs/source/downloader.rst: -------------------------------------------------------------------------------- 1 | downloader 2 | ========== 3 | 4 | .. py:module:: pydriosm.downloader 5 | 6 | .. automodule:: pydriosm.downloader 7 | :noindex: 8 | :no-members: 9 | :no-undoc-members: 10 | :no-inherited-members: 11 | 12 | Download OSM data 13 | ----------------- 14 | 15 | .. autosummary:: 16 | :toctree: _generated/ 17 | :template: class.rst 18 | 19 | GeofabrikDownloader 20 | BBBikeDownloader 21 | -------------------------------------------------------------------------------- /docs/source/errors.rst: -------------------------------------------------------------------------------- 1 | errors 2 | ====== 3 | 4 | .. py:module:: pydriosm.errors 5 | 6 | .. automodule:: pydriosm.errors 7 | :noindex: 8 | :no-members: 9 | :no-undoc-members: 10 | :no-inherited-members: 11 | 12 | Validation errors 13 | ----------------- 14 | 15 | .. autosummary:: 16 | :toctree: _generated/ 17 | :template: class.rst 18 | 19 | InvalidSubregionNameError 20 | InvalidFileFormatError 21 | 22 | Parse errors 23 | ------------ 24 | 25 | .. autosummary:: 26 | :toctree: _generated/ 27 | :template: class.rst 28 | 29 | OtherTagsReformatError 30 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | ######## 2 | PyDriosm 3 | ######## 4 | 5 | |PyPI| |Python Version| |Documentation Status| |License| |Codacy Badge| |DOI| 6 | 7 | .. |PyPI| image:: https://img.shields.io/pypi/v/pydriosm 8 | :alt: PyPI - Release 9 | :target: https://pypi.org/project/pydriosm/ 10 | .. |Python Version| image:: https://img.shields.io/pypi/pyversions/pydriosm 11 | :alt: PyPI - Python version 12 | :target: https://docs.python.org/3/ 13 | .. |Documentation Status| image:: https://readthedocs.org/projects/pydriosm/badge/?version=latest 14 | :alt: ReadTheDocs - Documentation status 15 | :target: https://pydriosm.readthedocs.io/en/latest/?badge=latest 16 | .. |License| image:: https://img.shields.io/pypi/l/pydriosm 17 | :alt: PyPI - License 18 | :target: https://github.com/mikeqfu/pydriosm/blob/master/LICENSE 19 | .. |Codacy Badge| image:: https://app.codacy.com/project/badge/Grade/b411ce89cbc445f58377a5799646d4cb 20 | :alt: Codacy grade (Code quality) 21 | :target: https://www.codacy.com/gh/mikeqfu/pydriosm/dashboard?utm_source=github.com&utm_medium=referral&utm_content=mikeqfu/pydriosm&utm_campaign=Badge_Grade 22 | .. |DOI| image:: https://zenodo.org/badge/92493726.svg 23 | :alt: Zenodo - DOI 24 | :target: https://zenodo.org/badge/latestdoi/92493726 25 | 26 | | **Author**: Qian Fu 27 | | **Email**: q.fu@bham.ac.uk 28 | 29 | PyDriosm is an open-source tool that provides an effortless way to download and access `OpenStreetMap `_ (OSM) data in popular file formats, such as `shapefile `_ and `protobuf binary format `_ (PBF), which are freely available from `Geofabrik `_ and `BBBike `_. Additionally, the package offers a comprehensive solution for convenient I/O operations and efficient storage capabilities for parsed OSM data within `PostgreSQL `_ databases. This means that users can easily read from and write to PostgreSQL databases, enabling efficient data manipulation, querying, and other essential tasks. Whether you are a researcher, practitioner, or simply interested in working with OSM data, PyDriosm can be useful and helpful to streamline your workflow and enhance your experience. 30 | 31 | 32 | .. toctree:: 33 | :maxdepth: 1 34 | :includehidden: 35 | :caption: Documentation 36 | 37 | installation 38 | modules 39 | license 40 | acknowledgement 41 | contributors 42 | 43 | .. toctree:: 44 | :maxdepth: 2 45 | :includehidden: 46 | :caption: Tutorial 47 | 48 | quick-start 49 | 50 | 51 | Indices 52 | ####### 53 | 54 | * :ref:`genindex` 55 | * :ref:`modindex` 56 | * :ref:`search` 57 | -------------------------------------------------------------------------------- /docs/source/installation.rst: -------------------------------------------------------------------------------- 1 | ============ 2 | Installation 3 | ============ 4 | 5 | To install the latest release of PyDriosm from `PyPI`_ via `pip`_: 6 | 7 | .. _`PyPI`: https://pypi.org/project/pydriosm/ 8 | .. _`pip`: https://pip.pypa.io/en/stable/cli/pip/ 9 | 10 | .. code-block:: console 11 | 12 | pip install --upgrade pydriosm 13 | 14 | 15 | To install the most recent version of PyDriosm hosted on `GitHub`_: 16 | 17 | .. _`GitHub`: https://github.com/mikeqfu/pydriosm 18 | 19 | .. code-block:: console 20 | 21 | pip install --upgrade git+https://github.com/mikeqfu/pydriosm.git 22 | 23 | 24 | .. warning:: 25 | 26 | - `Pip`_ may fail to install the dependency package `GDAL`_. In such a circumstance, try instead to `install their .whl files`_, which can be downloaded from the web page of the `archived "unofficial Windows binaries for Python extension packages"`_ (by Christoph Gohlke) or a `mirror site`_ (by Erin Turnbull). For how to install a .whl file, see the answers to this `StackOverflow question`_. 27 | 28 | .. _`GDAL`: https://pypi.org/project/GDAL/ 29 | .. _`archived "unofficial Windows binaries for Python extension packages"`: https://www.lfd.uci.edu/~gohlke/pythonlibs/ 30 | .. _`mirror site`: http://eturnbull.ca/pythonlibs/ 31 | .. _`StackOverflow question`: https://stackoverflow.com/questions/27885397 32 | 33 | 34 | .. note:: 35 | 36 | - If using a `virtual environment`_, make sure it is activated. 37 | - It is recommended to add `pip install`_ the option ``--upgrade`` (or ``-U``) to ensure that you are getting the latest stable release of the package. 38 | - Non-essential dependencies (e.g. `GeoPandas`_) of PyDriosm are not enforced to be installed along with the installation of the package. This is intended to optimise the installation requirements. If a `ModuleNotFoundError`_ or an `ImportError`_ pops out when importing/running a function or a method, first try to install the module(s)/package(s) mentioned in the error message, and then try to import/run the function or method again. 39 | - For more general instructions on the installation of Python packages, please refer to the official guide of `Installing Packages`_. 40 | 41 | .. _`virtual environment`: https://packaging.python.org/glossary/#term-Virtual-Environment 42 | .. _`pip install`: https://pip.pypa.io/en/stable/cli/pip_install/ 43 | .. _`ModuleNotFoundError`: https://docs.python.org/3/library/exceptions.html#ModuleNotFoundError 44 | .. _`ImportError`: https://docs.python.org/3/library/exceptions.html#ImportError 45 | .. _`GeoPandas`: https://geopandas.org/en/stable/getting_started/install.html#installing-with-pip 46 | .. _`install their .whl files`: https://stackoverflow.com/a/27909082/4981844 47 | .. _`Installing Packages`: https://packaging.python.org/tutorials/installing-packages/ 48 | 49 | 50 | To check whether PyDriosm has been correctly installed, try to import the package via an interpreter shell: 51 | 52 | .. code-block:: python 53 | :name: cmd current version 54 | 55 | >>> import pydriosm 56 | 57 | >>> pydriosm.__version__ # Check the latest version 58 | 59 | .. parsed-literal:: 60 | The latest version is: |version| -------------------------------------------------------------------------------- /docs/source/introduction.rst: -------------------------------------------------------------------------------- 1 | :orphan: 2 | 3 | ============== 4 | About PyDriosm 5 | ============== 6 | 7 | PyDriosm is an open-source tool that provides an effortless way to download and access `OpenStreetMap`_ (OSM) data in popular file formats, such as `shapefile`_ and `protobuf binary format`_ (PBF), which are freely available from `Geofabrik`_ and `BBBike`_. Additionally, the package offers a comprehensive solution for convenient I/O operations and efficient storage capabilities for parsed OSM data within `PostgreSQL`_ databases. This means that users can easily read from and write to PostgreSQL databases, enabling efficient data manipulation, querying, and other essential tasks. Whether you are a researcher, practitioner, or simply interested in working with OSM data, PyDriosm is a valuable tool to streamline your workflow and enhance your experience. 8 | 9 | .. _`OpenStreetMap`: https://www.openstreetmap.org/ 10 | .. _`shapefile`: https://wiki.openstreetmap.org/wiki/Shapefiles 11 | .. _`protobuf binary format`: https://wiki.openstreetmap.org/wiki/PBF_Format 12 | .. _`Geofabrik`: https://download.geofabrik.de/ 13 | .. _`BBBike`: https://download.bbbike.org/ 14 | .. _`PostgreSQL`: https://www.postgresql.org/ 15 | -------------------------------------------------------------------------------- /docs/source/ios.rst: -------------------------------------------------------------------------------- 1 | ios 2 | === 3 | 4 | .. py:module:: pydriosm.ios 5 | 6 | .. automodule:: pydriosm.ios 7 | :noindex: 8 | :no-members: 9 | :no-undoc-members: 10 | :no-inherited-members: 11 | 12 | Manage I/O storage of parsed OSM data 13 | ------------------------------------- 14 | 15 | .. autosummary:: 16 | :toctree: _generated/ 17 | :template: class.rst 18 | 19 | PostgresOSM 20 | 21 | Customised alternatives (Optional) 22 | ---------------------------------- 23 | 24 | .. autosummary:: 25 | :toctree: _generated/ 26 | :template: class2.rst 27 | 28 | GeofabrikIOS 29 | BBBikeIOS 30 | 31 | Other utilities 32 | --------------- 33 | 34 | .. py:module:: pydriosm.ios.utils 35 | 36 | .. automodule:: pydriosm.ios.utils 37 | :noindex: 38 | :no-members: 39 | :no-undoc-members: 40 | :no-inherited-members: 41 | 42 | .. autosummary:: 43 | :toctree: _generated/ 44 | :template: function.rst 45 | 46 | get_default_layer_name 47 | validate_schema_names 48 | validate_table_name 49 | -------------------------------------------------------------------------------- /docs/source/latexindex.rst: -------------------------------------------------------------------------------- 1 | :orphan: 2 | 3 | ###################### 4 | PyDriosm Documentation 5 | ###################### 6 | 7 | .. toctree:: 8 | :maxdepth: 1 9 | :includehidden: 10 | :caption: Documentation 11 | 12 | introduction 13 | installation 14 | modules 15 | license 16 | acknowledgement 17 | contributors 18 | 19 | .. toctree:: 20 | :maxdepth: 2 21 | :includehidden: 22 | :caption: Tutorial 23 | 24 | quick-start 25 | -------------------------------------------------------------------------------- /docs/source/license.rst: -------------------------------------------------------------------------------- 1 | ======= 2 | License 3 | ======= 4 | 5 | - **PyDriosm** 6 | 7 | - `PyDriosm `_ is licensed under `GNU General Public License v3.0 `_ or later (GPLv3+). 8 | 9 | - **OpenStreetMap data** 10 | 11 | - The free `OpenStreetMap `_ data, which is used for the development of PyDriosm, is licensed under the `Open Data Commons Open Database License `_ (ODbL) by the `OpenStreetMap Foundation `_ (OSMF). 12 | - For more details about the use of the OpenStreetMap data, refer to the web page of `Copyright and Licence `_. 13 | -------------------------------------------------------------------------------- /docs/source/modules.rst: -------------------------------------------------------------------------------- 1 | ====================== 2 | Sub-packages / modules 3 | ====================== 4 | 5 | .. py:module:: pydriosm 6 | 7 | The package includes the following sub-packages and modules: 8 | 9 | Sub-packages 10 | ============ 11 | 12 | .. autosummary:: 13 | 14 | downloader 15 | reader 16 | ios 17 | 18 | .. toctree:: 19 | :maxdepth: 2 20 | :hidden: 21 | 22 | downloader 23 | reader 24 | ios 25 | 26 | Modules 27 | ======= 28 | 29 | .. autosummary:: 30 | 31 | errors 32 | utils 33 | 34 | .. toctree:: 35 | :maxdepth: 2 36 | :hidden: 37 | 38 | errors 39 | utils 40 | -------------------------------------------------------------------------------- /docs/source/reader.rst: -------------------------------------------------------------------------------- 1 | reader 2 | ====== 3 | 4 | .. py:module:: pydriosm.reader 5 | 6 | .. automodule:: pydriosm.reader 7 | :noindex: 8 | :no-members: 9 | :no-undoc-members: 10 | :no-inherited-members: 11 | 12 | 13 | Transform OSM data 14 | ------------------ 15 | 16 | .. autosummary:: 17 | :toctree: _generated/ 18 | :template: class.rst 19 | 20 | Transformer 21 | 22 | Parse OSM data 23 | -------------- 24 | 25 | .. autosummary:: 26 | :toctree: _generated/ 27 | :template: class.rst 28 | 29 | SHPReadParse 30 | PBFReadParse 31 | VarReadParse 32 | 33 | Read OSM data 34 | ------------- 35 | 36 | .. autosummary:: 37 | :toctree: _generated/ 38 | :template: class.rst 39 | 40 | GeofabrikReader 41 | BBBikeReader 42 | -------------------------------------------------------------------------------- /docs/source/rtd-req.txt: -------------------------------------------------------------------------------- 1 | pyhelpers 2 | pyrcs 3 | pyshp 4 | Sphinx==7.2.6 5 | sphinx-copybutton==0.5.2 6 | sphinx-rtd-theme==1.3.0 7 | -------------------------------------------------------------------------------- /docs/source/utils.rst: -------------------------------------------------------------------------------- 1 | utils 2 | ===== 3 | 4 | .. py:module:: pydriosm.utils 5 | 6 | .. automodule:: pydriosm.utils 7 | :noindex: 8 | :no-members: 9 | :no-undoc-members: 10 | :no-inherited-members: 11 | 12 | Check data pathnames 13 | -------------------- 14 | 15 | .. autosummary:: 16 | :toctree: _generated/ 17 | :template: function.rst 18 | 19 | check_relpath 20 | cdd_geofabrik 21 | cdd_bbbike 22 | 23 | General utilities 24 | ----------------- 25 | 26 | .. autosummary:: 27 | :toctree: _generated/ 28 | :template: function.rst 29 | 30 | first_unique 31 | check_json_engine 32 | remove_osm_file 33 | -------------------------------------------------------------------------------- /pydriosm/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Package initialization. 3 | """ 4 | 5 | import datetime 6 | import json 7 | import pkgutil 8 | 9 | from .downloader import BBBikeDownloader, GeofabrikDownloader 10 | from .ios import PostgresOSM 11 | from .reader import BBBikeReader, GeofabrikReader, PBFReadParse, SHPReadParse, VarReadParse 12 | 13 | metadata = json.loads(pkgutil.get_data(__name__, "data/metadata.json").decode()) 14 | 15 | __project__ = metadata['Project'] 16 | __pkgname__ = metadata['Package'] 17 | 18 | __author__ = metadata['Author'] 19 | __affiliation__ = metadata['Affiliation'] 20 | __author_email__ = metadata['Email'] 21 | 22 | __description__ = metadata['Description'] 23 | 24 | __copyright__ = f'2019-{datetime.datetime.now().year}, {__author__}' 25 | 26 | __version__ = metadata['Version'] 27 | __license__ = metadata['License'] 28 | 29 | __first_release_date__ = metadata['First release'] 30 | 31 | __all__ = [ 32 | __author__, 33 | __description__, 34 | __author_email__, 35 | __pkgname__, 36 | __version__, 37 | 'downloader', 'GeofabrikDownloader', 'BBBikeDownloader', 38 | 'reader', 'PBFReadParse', 'SHPReadParse', 'VarReadParse', 'GeofabrikReader', 'BBBikeReader', 39 | 'ios', 'PostgresOSM', 40 | ] 41 | -------------------------------------------------------------------------------- /pydriosm/_updater.py: -------------------------------------------------------------------------------- 1 | """ 2 | Update (prepacked) package data. 3 | """ 4 | 5 | import time 6 | 7 | from pyhelpers.ops import confirmed 8 | 9 | from .downloader import BBBikeDownloader, GeofabrikDownloader 10 | 11 | 12 | def _update_prepacked_data(verbose=True, interval=5): 13 | """ 14 | Update prepacked data used by the downloader classes. 15 | 16 | :param verbose: whether to print relevant information in console, defaults to ``True`` 17 | :type verbose: bool | int 18 | :param interval: time gap (in seconds) between the updating of different classes, 19 | defaults to ``5`` (seconds) 20 | :type interval: int | float 21 | 22 | **Examples**:: 23 | 24 | >>> from pydriosm._updater import _update_prepacked_data 25 | 26 | >>> _update_prepacked_data(verbose=True) 27 | To update resources (which may take a few minutes) 28 | ? [No]|Yes: no 29 | """ 30 | 31 | if confirmed("To update resources (which may take a few minutes)\n?"): 32 | 33 | meth_args = { 34 | 'update': True, 35 | 'confirmation_required': False, 36 | 'verbose': verbose, 37 | } 38 | 39 | # -- Geofabrik ----------------------------------------------------------------------------- 40 | gfd = GeofabrikDownloader() 41 | 42 | _ = gfd.get_download_index(**meth_args) 43 | 44 | time.sleep(interval) 45 | 46 | _ = gfd.get_continent_tables(**meth_args) 47 | 48 | time.sleep(interval) 49 | 50 | _ = gfd.get_region_subregion_tier(**meth_args) 51 | 52 | time.sleep(interval) 53 | 54 | _ = gfd.get_catalogue(**meth_args) 55 | 56 | time.sleep(interval) 57 | 58 | _ = gfd.get_valid_subregion_names(**meth_args) 59 | 60 | time.sleep(interval) 61 | 62 | # -- BBBike -------------------------------------------------------------------------------- 63 | bbd = BBBikeDownloader() 64 | 65 | _ = bbd.get_names_of_cities(**meth_args) 66 | 67 | time.sleep(interval) 68 | 69 | _ = bbd.get_coordinates_of_cities(**meth_args) 70 | 71 | time.sleep(interval) 72 | 73 | _ = bbd.get_subregion_index(**meth_args) 74 | 75 | time.sleep(interval) 76 | 77 | _ = bbd.get_valid_subregion_names(**meth_args) 78 | 79 | time.sleep(interval) 80 | 81 | _ = bbd.get_catalogue(**meth_args) 82 | 83 | if verbose: 84 | print("\nUpdate finished.") 85 | -------------------------------------------------------------------------------- /pydriosm/data/bbbike_cities.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/pydriosm/data/bbbike_cities.pkl -------------------------------------------------------------------------------- /pydriosm/data/bbbike_cities_coordinates.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/pydriosm/data/bbbike_cities_coordinates.pkl -------------------------------------------------------------------------------- /pydriosm/data/bbbike_downloads_catalogue.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/pydriosm/data/bbbike_downloads_catalogue.pkl -------------------------------------------------------------------------------- /pydriosm/data/bbbike_index_of_subregions.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/pydriosm/data/bbbike_index_of_subregions.pkl -------------------------------------------------------------------------------- /pydriosm/data/bbbike_subregion_names.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/pydriosm/data/bbbike_subregion_names.pkl -------------------------------------------------------------------------------- /pydriosm/data/geofabrik_continent_tables.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/pydriosm/data/geofabrik_continent_tables.pkl -------------------------------------------------------------------------------- /pydriosm/data/geofabrik_downloads_catalogue.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/pydriosm/data/geofabrik_downloads_catalogue.pkl -------------------------------------------------------------------------------- /pydriosm/data/geofabrik_index_of_subregions.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/pydriosm/data/geofabrik_index_of_subregions.pkl -------------------------------------------------------------------------------- /pydriosm/data/geofabrik_region-subregion_tier.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/pydriosm/data/geofabrik_region-subregion_tier.pkl -------------------------------------------------------------------------------- /pydriosm/data/geofabrik_subregion_names.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/pydriosm/data/geofabrik_subregion_names.pkl -------------------------------------------------------------------------------- /pydriosm/data/metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "Project": "PyDriosm", 3 | "Package": "pydriosm", 4 | "Author": "Qian Fu", 5 | "Affiliation": "Birmingham Centre for Railway Research and Education, School of Engineering, University of Birmingham", 6 | "Email": "q.fu@bham.ac.uk", 7 | "Description": "An open-source tool for downloading, reading and PostgreSQL-based I/O of OpenStreetMap data.", 8 | "Version": "2.3.0b1", 9 | "License": "GPLv3+", 10 | "First release": "September 2019" 11 | } -------------------------------------------------------------------------------- /pydriosm/downloader/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Download `OpenStreetMap `_ (OSM) data from free download servers: 3 | `Geofabrik `_ and `BBBike `_. 4 | """ 5 | 6 | from .bbbike import BBBikeDownloader 7 | from .geofabrik import GeofabrikDownloader 8 | 9 | __all__ = [ 10 | 'GeofabrikDownloader', 11 | 'BBBikeDownloader', 12 | ] 13 | -------------------------------------------------------------------------------- /pydriosm/errors.py: -------------------------------------------------------------------------------- 1 | """ 2 | Define custom errors/exceptions. 3 | """ 4 | 5 | 6 | class InvalidSubregionNameError(Exception): 7 | """ 8 | Exception raised when an input `subregion_name` is not recognizable. 9 | """ 10 | 11 | def __init__(self, subregion_name, msg=None): 12 | """ 13 | :param subregion_name: name of a (sub)region available on a free download server 14 | :type subregion_name: str 15 | :param msg: index of optional messages, defaults to ``None``; options include {1, 2} 16 | :type msg: int | None 17 | 18 | :ivar: str subregion_name: name of a (sub)region available on a free download server 19 | :ivar: int | None msg: index of optional messages; options include {1, 2} 20 | :ivar: str: error message 21 | 22 | **Examples**:: 23 | 24 | >>> from pydriosm.errors import InvalidSubregionNameError 25 | 26 | >>> raise InvalidSubregionNameError(subregion_name='abc') 27 | Traceback (most recent call last): 28 | ... 29 | pydriosm.errors.InvalidSubregionNameError: 30 | `subregion_name='abc'` -> The input of `subregion_name` is not recognizable. 31 | Check the `.data_source`, or try another one instead. 32 | 33 | >>> from pydriosm.downloader import GeofabrikDownloader, BBBikeDownloader 34 | 35 | >>> gfd = GeofabrikDownloader() 36 | >>> gfd.validate_subregion_name(subregion_name='birmingham') 37 | Traceback (most recent call last): 38 | ... 39 | pydriosm.errors.InvalidSubregionNameError: 40 | `subregion_name='birmingham'` 41 | 1) `subregion_name` fails to match any in `.valid_subregion_names`; or 42 | 2) The queried (sub)region is not available on the free download server. 43 | 44 | >>> bbd = BBBikeDownloader() 45 | >>> bbd.validate_subregion_name(subregion_name='bham') 46 | Traceback (most recent call last): 47 | ... 48 | pydriosm.errors.InvalidSubregionNameError: 49 | `subregion_name='bham'` -> The input of `subregion_name` is not recognizable. 50 | Check the `.data_source`, or try another one instead. 51 | """ 52 | 53 | self.subregion_name = subregion_name 54 | self.msg = msg 55 | 56 | if self.msg == 1: 57 | self.message = \ 58 | "\t1) `subregion_name` fails to match any in " \ 59 | "`.valid_subregion_names`; " \ 60 | "or\n" \ 61 | "\t2) The queried (sub)region is not available on the free download server." 62 | else: 63 | self.message = "The input of `subregion_name` is not recognizable.\n" \ 64 | " Check the `.data_source`, or try another one instead." 65 | 66 | super().__init__(self.message) 67 | 68 | def __str__(self): 69 | conj = "\n" if self.msg == 1 else " -> " 70 | return f"\n `subregion_name='{self.subregion_name}'`{conj}{self.message}" 71 | 72 | 73 | class InvalidFileFormatError(Exception): 74 | """ 75 | Exception raised when an input `osm_file_format` is not recognizable. 76 | """ 77 | 78 | def __init__(self, osm_file_format, valid_file_formats=None): 79 | """ 80 | :param osm_file_format: file format/extension of the OSM data on the free download server 81 | :type osm_file_format: str 82 | :param valid_file_formats: filename extensions of the data files available on 83 | the free download server, defaults to ``None`` 84 | :type valid_file_formats: typing.Iterable | None 85 | 86 | :ivar: str osm_file_format: file format/extension of the OSM data 87 | available on the free download server 88 | :ivar: int | None message: error message 89 | 90 | **Examples**:: 91 | 92 | >>> from pydriosm.errors import InvalidFileFormatError 93 | 94 | >>> raise InvalidFileFormatError(osm_file_format='abc') 95 | Traceback (most recent call last): 96 | ... 97 | pydriosm.errors.InvalidFileFormatError: 98 | `osm_file_format='abc'` -> The input `osm_file_format` is unidentifiable. 99 | 100 | >>> from pydriosm.downloader import GeofabrikDownloader, BBBikeDownloader 101 | 102 | >>> gfd = GeofabrikDownloader() 103 | >>> gfd.validate_file_format(osm_file_format='abc') 104 | Traceback (most recent call last): 105 | ... 106 | pydriosm.errors.InvalidFileFormatError: 107 | `osm_file_format='abc'` -> The input `osm_file_format` is unidentifiable. 108 | Valid options include: {'.shp.zip', '.osm.pbf', '.osm.bz2'}. 109 | 110 | >>> bbd = BBBikeDownloader() 111 | >>> bbd.validate_file_format(osm_file_format='abc') 112 | Traceback (most recent call last): 113 | ... 114 | pydriosm.errors.InvalidFileFormatError: 115 | `osm_file_format='abc'` -> The input `osm_file_format` is unidentifiable. 116 | Valid options include: {'.shp.zip', '.geojson.xz', '.mapsforge-osm.zip', '.pbf', ... 117 | """ 118 | 119 | self.osm_file_format = osm_file_format 120 | 121 | self.message = "The input `osm_file_format` is unidentifiable." 122 | if valid_file_formats: 123 | self.message += f"\n\tValid options include: {valid_file_formats}." 124 | 125 | super().__init__(self.message) 126 | 127 | def __str__(self): 128 | return f"\n `osm_file_format='{self.osm_file_format}'` -> {self.message}" 129 | 130 | 131 | class OtherTagsReformatError(Exception): 132 | """ 133 | Exception raised when errors occur in the process of parsing ``other_tags`` in a PBF data file. 134 | """ 135 | 136 | def __init__(self, other_tags): 137 | """ 138 | :param other_tags: data of ``'other_tags'`` of a single feature in a PBF data file 139 | :type other_tags: str | None 140 | 141 | :ivar str | None other_tags: data of ``'other_tags'`` of a single feature 142 | in a PBF data file 143 | :ivar str message: error message 144 | 145 | **Examples**:: 146 | 147 | >>> from pydriosm.errors import OtherTagsReformatError 148 | 149 | >>> raise OtherTagsReformatError(other_tags='abc') 150 | Traceback (most recent call last): 151 | ... 152 | pydriosm.errors.OtherTagsReformatError: 153 | `other_tags='abc'` -> Failed to reformat the ... 154 | """ 155 | 156 | self.other_tags = other_tags 157 | self.message = "Failed to reformat the `other_tags`." 158 | 159 | super().__init__(self.message) 160 | 161 | def __str__(self): 162 | return f"\n `other_tags='{self.other_tags}'` -> {self.message}" 163 | -------------------------------------------------------------------------------- /pydriosm/ios/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Implement storage I/O of (parsed) OSM data extracts with `PostgreSQL `_. 3 | """ 4 | 5 | from ._ios import PostgresOSM 6 | from .bbbike import BBBikeIOS 7 | from .geofabrik import GeofabrikIOS 8 | 9 | __all__ = ['PostgresOSM', 'GeofabrikIOS', 'BBBikeIOS'] 10 | -------------------------------------------------------------------------------- /pydriosm/ios/bbbike.py: -------------------------------------------------------------------------------- 1 | """ 2 | Implement storage I/O of (parsed) OSM data extracts (available from BBBike free download server) 3 | with `PostgreSQL `_. 4 | """ 5 | 6 | from pydriosm.ios._ios import PostgresOSM 7 | 8 | 9 | class BBBikeIOS: 10 | """ 11 | Implement storage I/O of `BBBike exports of OpenStreetMap data `_ 12 | with PostgreSQL. 13 | """ 14 | 15 | def __init__(self, **kwargs): 16 | """ 17 | :param kwargs: [optional] parameters of the class :class:`~pydriosm.downloader.PostgresOSM` 18 | 19 | :ivar BBBikeDownloader downloader: instance of the class 20 | :class:`~pydriosm.downloader.BBBikeDownloader` 21 | :ivar BBBikeReader reader: instance of the class 22 | :class:`~pydriosm.downloader.BBBikeReader` 23 | 24 | **Examples**:: 25 | 26 | >>> from pydriosm.ios import BBBikeIOS 27 | 28 | >>> bbi = BBBikeIOS(database_name='osmdb_test') 29 | Password (postgres@localhost:5432): *** 30 | Creating a database: "osmdb_test" ... Done. 31 | Connecting postgres:***@localhost:5432/osmdb_test ... Successfully. 32 | 33 | >>> type(bbi.dbms) 34 | pydriosm.ios.PostgresOSM 35 | 36 | >>> bbi.dbms.name 37 | 'BBBike exports of OpenStreetMap data' 38 | 39 | .. seealso:: 40 | 41 | - Examples for all the methods of the class :class:`~pydriosm.ios.PostgresOSM`. 42 | """ 43 | 44 | kwargs.update({'data_source': 'BBBike'}) 45 | self.dbms = PostgresOSM(**kwargs) 46 | -------------------------------------------------------------------------------- /pydriosm/ios/geofabrik.py: -------------------------------------------------------------------------------- 1 | """ 2 | Implement storage I/O of (parsed) OSM data extracts (available from Geofabrik free download server) 3 | with `PostgreSQL `_. 4 | """ 5 | 6 | from pydriosm.ios._ios import PostgresOSM 7 | 8 | 9 | class GeofabrikIOS: 10 | """ 11 | Implement storage I/O of `Geofabrik OpenStreetMap data extracts `_ 12 | with `PostgreSQL `_. 13 | """ 14 | 15 | def __init__(self, **kwargs): 16 | """ 17 | :param kwargs: [optional] parameters of the class :class:`~pydriosm.downloader.PostgresOSM` 18 | 19 | :ivar PostgresOSM postgres: instance of the class :class:`~pydriosm.downloader.PostgresOSM` 20 | :ivar GeofabrikDownloader downloader: instance of the class 21 | :class:`~pydriosm.downloader.GeofabrikDownloader` 22 | :ivar GeofabrikReader reader: instance of the class 23 | :class:`~pydriosm.downloader.GeofabrikReader` 24 | 25 | **Examples**:: 26 | 27 | >>> from pydriosm.ios import GeofabrikIOS 28 | 29 | >>> gfi = GeofabrikIOS(database_name='osmdb_test') 30 | Password (postgres@localhost:5432): *** 31 | Creating a database: "osmdb_test" ... Done. 32 | Connecting postgres:***@localhost:5432/osmdb_test ... Successfully. 33 | 34 | >>> type(gfi.dbms) 35 | pydriosm.ios.PostgresOSM 36 | 37 | >>> gfi.dbms.name 38 | 'Geofabrik OpenStreetMap data extracts' 39 | 40 | .. seealso:: 41 | 42 | - Examples for all the methods of the class :class:`~pydriosm.ios.PostgresOSM`. 43 | """ 44 | 45 | kwargs.update({'data_source': 'Geofabrik'}) 46 | self.dbms = PostgresOSM(**kwargs) 47 | -------------------------------------------------------------------------------- /pydriosm/ios/utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | Utilities for the :mod:`~pydriosm.ios` module. 3 | """ 4 | 5 | from pyhelpers.text import find_similar_str, remove_punctuation 6 | 7 | from pydriosm.reader import PBFReadParse, SHPReadParse 8 | 9 | 10 | def get_default_layer_name(schema_name): 11 | """ 12 | Get default name (as an input schema name) of an OSM layer 13 | for the class :py:class:`PostgresOSM`. 14 | 15 | See, for example, the method :meth:`pydriosm.ios.PostgresOSM.import_osm_layer`. 16 | 17 | :param schema_name: name of a schema (or name of an OSM layer) 18 | :type schema_name: str 19 | :return: default name of the layer 20 | :rtype: str 21 | 22 | **Examples**:: 23 | 24 | >>> from pydriosm.ios import get_default_layer_name 25 | 26 | >>> lyr_name = get_default_layer_name(schema_name='point') 27 | >>> lyr_name 28 | 'points' 29 | 30 | >>> lyr_name = get_default_layer_name(schema_name='land') 31 | >>> lyr_name 32 | 'landuse' 33 | """ 34 | 35 | valid_layer_names = set(PBFReadParse.LAYER_GEOM.keys()).union(SHPReadParse.LAYER_NAMES) 36 | 37 | layer_name_ = find_similar_str(x=schema_name, lookup_list=valid_layer_names) 38 | 39 | return layer_name_ 40 | 41 | 42 | def validate_schema_names(schema_names=None, schema_named_as_layer=False): 43 | """ 44 | Validate schema names for importing data into a database. 45 | 46 | :param schema_names: one or multiple names of layers, e.g. 'points', 'lines', defaults to ``None`` 47 | :type schema_names: typing.Iterable | None 48 | :param schema_named_as_layer: whether to use default PBF layer name as the schema name, 49 | defaults to ``False`` 50 | :type schema_named_as_layer: bool 51 | :return: valid names of the schemas in the database 52 | :rtype: list 53 | 54 | **Examples**:: 55 | 56 | >>> from pydriosm.ios import validate_schema_names 57 | 58 | >>> valid_names = validate_schema_names() 59 | >>> valid_names 60 | [] 61 | 62 | >>> input_schema_names = ['point', 'polygon'] 63 | >>> valid_names = validate_schema_names(input_schema_names) 64 | >>> valid_names 65 | ['point', 'polygon'] 66 | 67 | >>> valid_names = validate_schema_names(input_schema_names, schema_named_as_layer=True) 68 | >>> valid_names 69 | ['points', 'multipolygons'] 70 | """ 71 | 72 | if schema_names: 73 | if isinstance(schema_names, str): 74 | schema_names_ = [ 75 | get_default_layer_name(schema_names) if schema_named_as_layer else schema_names] 76 | # assert schema_names_[0] in valid_layer_names, assertion_msg 77 | else: # isinstance(schema_names, list) is True 78 | if schema_named_as_layer: 79 | schema_names_ = [get_default_layer_name(x) for x in schema_names] 80 | else: 81 | schema_names_ = schema_names 82 | else: 83 | schema_names_ = [] 84 | 85 | return schema_names_ 86 | 87 | 88 | def validate_table_name(table_name, sub_space=''): 89 | """ 90 | Validate a table name for importing OSM data into a database. 91 | 92 | :param table_name: name as input of a table in a PostgreSQL database 93 | :type table_name: str 94 | :param sub_space: substitute for space, defaults to ``''`` 95 | :type sub_space: str 96 | :return: valid name of the table in the database 97 | :rtype: str 98 | 99 | **Examples**:: 100 | 101 | >>> from pydriosm.ios import validate_table_name 102 | 103 | >>> subrgn_name = 'greater london' 104 | >>> valid_table_name = validate_table_name(subrgn_name) 105 | >>> valid_table_name 106 | 'greater london' 107 | 108 | >>> subrgn_name = 'Llanfairpwllgwyngyllgogerychwyrndrobwllllantysiliogogogoch, Wales' 109 | >>> valid_table_name = validate_table_name(subrgn_name, sub_space='_') 110 | >>> valid_table_name 111 | 'Llanfairpwllgwyngyllgogerychwyrndrobwllllantysiliogogogoch_W..' 112 | """ 113 | 114 | table_name_ = remove_punctuation(x=table_name, rm_whitespace=True) 115 | 116 | if sub_space: 117 | table_name_ = table_name_.replace(' ', sub_space) 118 | 119 | table_name_ = table_name_[:60] + '..' if len(table_name_) >= 63 else table_name_ 120 | 121 | return table_name_ 122 | -------------------------------------------------------------------------------- /pydriosm/reader/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Read the OSM data extracts in various file formats. 3 | """ 4 | 5 | from .bbbike import BBBikeReader 6 | from .geofabrik import GeofabrikReader 7 | from .parser import PBFReadParse, SHPReadParse, VarReadParse 8 | from .transformer import Transformer 9 | 10 | __all__ = [ 11 | 'GeofabrikReader', 'BBBikeReader', 12 | 'Transformer', 13 | 'PBFReadParse', 'SHPReadParse', 'VarReadParse' 14 | ] 15 | -------------------------------------------------------------------------------- /pydriosm/reader/bbbike.py: -------------------------------------------------------------------------------- 1 | """ 2 | Read OpenStreetMap data extracts available from BBBike free download server. 3 | """ 4 | 5 | from pydriosm.downloader import BBBikeDownloader 6 | from pydriosm.reader._reader import _Reader 7 | 8 | 9 | class BBBikeReader(_Reader): 10 | """ 11 | Read `BBBike `_ exports of OpenStreetMap data. 12 | """ 13 | 14 | #: str: Default download directory. 15 | DEFAULT_DOWNLOAD_DIR = "osm_data\\bbbike" 16 | #: set: Valid file formats. 17 | FILE_FORMATS = { 18 | '.csv.xz', 19 | '.garmin-onroad-latin1.zip', 20 | '.garmin-onroad.zip', 21 | '.garmin-opentopo.zip', 22 | '.garmin-osm.zip', 23 | '.geojson.xz', 24 | '.gz', 25 | '.mapsforge-osm.zip', 26 | '.pbf', 27 | '.shp.zip', 28 | '.svg-osm.zip', 29 | } 30 | 31 | def __init__(self, data_dir=None, max_tmpfile_size=None): 32 | """ 33 | :param data_dir: (a path or a name of) a directory where a data file is; 34 | if ``None`` (default), a folder ``osm_bbbike`` under the current working directory 35 | :type data_dir: str | None 36 | :param max_tmpfile_size: defaults to ``None``, 37 | see also :func:`gdal_configurations` 38 | :type max_tmpfile_size: int | None 39 | 40 | :ivar BBBikeDownloader downloader: instance of the class 41 | :py:class:`BBBikeDownloader` 42 | :ivar str name: name of the data resource 43 | :ivar str url: url of the homepage to the BBBike free download server 44 | 45 | **Examples**:: 46 | 47 | >>> from pydriosm.reader import BBBikeReader 48 | 49 | >>> bbr = BBBikeReader() 50 | 51 | >>> bbr.NAME 52 | 'BBBike' 53 | """ 54 | 55 | # noinspection PyTypeChecker 56 | super().__init__( 57 | downloader=BBBikeDownloader, data_dir=data_dir, max_tmpfile_size=max_tmpfile_size) 58 | 59 | def read_osm_pbf(self, subregion_name, data_dir=None, readable=False, expand=False, 60 | parse_geometry=False, parse_other_tags=False, parse_properties=False, 61 | update=False, download=True, pickle_it=False, ret_pickle_path=False, 62 | rm_pbf_file=False, chunk_size_limit=50, verbose=False, **kwargs): 63 | """ 64 | Read a PBF (.osm.pbf) data file of a geographic (sub)region. 65 | 66 | :param subregion_name: name of a geographic (sub)region (case-insensitive) 67 | that is available on Geofabrik free download server 68 | :type subregion_name: str 69 | :param data_dir: directory where the .osm.pbf data file is located/saved; 70 | if ``None``, the default local directory 71 | :type data_dir: str | None 72 | :param readable: whether to parse each feature in the raw data, defaults to ``False`` 73 | :type readable: bool 74 | :param expand: whether to expand dict-like data into separate columns, defaults to ``False`` 75 | :type expand: bool 76 | :param parse_geometry: whether to represent the ``'geometry'`` field 77 | in a `shapely.geometry`_ format, defaults to ``False`` 78 | :type parse_geometry: bool 79 | :param parse_properties: whether to represent the ``'properties'`` field 80 | in a tabular format, defaults to ``False`` 81 | :type parse_properties: bool 82 | :param parse_other_tags: whether to represent a ``'other_tags'`` (of ``'properties'``) 83 | in a `dict`_ format, defaults to ``False`` 84 | :type parse_other_tags: bool 85 | :param download: whether to download/update the PBF data file of the given subregion, 86 | if it is not available at the specified path, defaults to ``True`` 87 | :type download: bool 88 | :param update: whether to check to update pickle backup (if available), defaults to ``False`` 89 | :type update: bool 90 | :param pickle_it: whether to save the .pbf data as a pickle file, defaults to ``False`` 91 | :type pickle_it: bool 92 | :param ret_pickle_path: (when ``pickle_it=True``) 93 | whether to return a path to the saved pickle file 94 | :type ret_pickle_path: bool 95 | :param rm_pbf_file: whether to delete the downloaded .osm.pbf file, defaults to ``False`` 96 | :type rm_pbf_file: bool 97 | :param chunk_size_limit: threshold (in MB) that triggers the use of chunk parser, 98 | defaults to ``50``; 99 | if the size of the .osm.pbf file (in MB) is greater than ``chunk_size_limit``, 100 | it will be parsed in a chunk-wise way 101 | :type chunk_size_limit: int | None 102 | :param verbose: whether to print relevant information in console as the function runs, 103 | defaults to ``False`` 104 | :type verbose: bool | int 105 | :param kwargs: [optional] parameters of the method 106 | :meth:`_Reader.read_osm_pbf()` 107 | :return: dictionary of the .osm.pbf data; 108 | when ``pickle_it=True``, return a tuple of the dictionary and a path to the pickle file 109 | :rtype: dict | tuple | None 110 | 111 | .. _`shapely.geometry`: 112 | https://shapely.readthedocs.io/en/latest/manual.html#geometric-objects 113 | .. _`dict`: 114 | https://docs.python.org/3/library/stdtypes.html#dict 115 | 116 | **Examples**:: 117 | 118 | >>> from pydriosm.reader import BBBikeReader 119 | >>> from pyhelpers.dirs import delete_dir 120 | 121 | >>> bbr = BBBikeReader() 122 | 123 | >>> subrgn_name = 'Leeds' 124 | >>> dat_dir = "tests\\osm_data" 125 | 126 | >>> leeds_pbf_raw = bbr.read_osm_pbf(subrgn_name, data_dir=dat_dir, verbose=True) 127 | Downloading "Leeds.osm.pbf" 128 | to "tests\\osm_data\\leeds\\" ... Done. 129 | Reading "tests\\osm_data\\leeds\\Leeds.osm.pbf" ... Done. 130 | >>> type(leeds_pbf_raw) 131 | dict 132 | >>> list(leeds_pbf_raw.keys()) 133 | ['points', 'lines', 'multilinestrings', 'multipolygons', 'other_relations'] 134 | 135 | >>> pbf_raw_points = leeds_pbf_raw['points'] 136 | >>> type(pbf_raw_points) 137 | list 138 | >>> type(pbf_raw_points[0]) 139 | osgeo.ogr.Feature 140 | 141 | >>> # (Parsing the data in this example might take up to a few minutes.) 142 | >>> leeds_pbf_parsed = bbr.read_osm_pbf( 143 | ... subrgn_name, data_dir=dat_dir, readable=True, expand=True, 144 | ... parse_geometry=True, parse_other_tags=True, parse_properties=True, 145 | ... verbose=True) 146 | Parsing "tests\\osm_data\\leeds\\Leeds.osm.pbf" ... Done. 147 | 148 | >>> list(leeds_pbf_parsed.keys()) 149 | ['points', 'lines', 'multilinestrings', 'multipolygons', 'other_relations'] 150 | 151 | >>> # Data of the 'multipolygons' layer 152 | >>> leeds_pbf_parsed_multipolygons = leeds_pbf_parsed['multipolygons'] 153 | >>> leeds_pbf_parsed_multipolygons.head() 154 | id geometry ... tourism other_tags 155 | 0 10595 (POLYGON ((-1.5030223 53.6725382, -1.5034495 5... ... None None 156 | 1 10600 (POLYGON ((-1.5116994 53.6764287, -1.5099361 5... ... None None 157 | 2 10601 (POLYGON ((-1.5142403 53.6710831, -1.5143686 5... ... None None 158 | 3 10612 (POLYGON ((-1.5129341 53.6704885, -1.5131883 5... ... None None 159 | 4 10776 (POLYGON ((-1.5523801 53.7029081, -1.5524772 5... ... None None 160 | [5 rows x 26 columns] 161 | 162 | >>> # Delete the example data and the test data directory 163 | >>> delete_dir(dat_dir, verbose=True) 164 | To delete the directory "tests\\osm_data\\" (Not empty) 165 | ? [No]|Yes: yes 166 | Deleting "tests\\osm_data\\" ... Done. 167 | 168 | .. seealso:: 169 | 170 | - Examples for the method 171 | :meth:`GeofabrikReader.read_osm_pbf()`. 172 | """ 173 | 174 | osm_pbf_data = super().read_osm_pbf( 175 | subregion_name=subregion_name, data_dir=data_dir, readable=readable, expand=expand, 176 | parse_geometry=parse_geometry, parse_properties=parse_properties, 177 | parse_other_tags=parse_other_tags, update=update, download=download, 178 | pickle_it=pickle_it, ret_pickle_path=ret_pickle_path, rm_pbf_file=rm_pbf_file, 179 | chunk_size_limit=chunk_size_limit, verbose=verbose, **kwargs) 180 | 181 | return osm_pbf_data 182 | 183 | def read_shp_zip(self, subregion_name, layer_names=None, feature_names=None, data_dir=None, 184 | update=False, download=True, pickle_it=False, ret_pickle_path=False, 185 | rm_extracts=False, rm_shp_zip=False, verbose=False, **kwargs): 186 | """ 187 | Read a shapefile of a geographic (sub)region. 188 | 189 | :param subregion_name: name of a geographic (sub)region (case-insensitive) 190 | that is available on BBBike free download server 191 | :type subregion_name: str 192 | :param layer_names: name of a .shp layer, e.g. 'railways', or names of multiple layers; 193 | if ``None`` (default), all available layers 194 | :type layer_names: str | list | None 195 | :param feature_names: name of a feature, e.g. 'rail', or names of multiple features; 196 | if ``None`` (default), all available features 197 | :type feature_names: str | list | None 198 | :param data_dir: directory where the .shp.zip data file is located/saved; 199 | if ``None``, the default directory 200 | :type data_dir: str | None 201 | :param update: whether to check to update pickle backup (if available), defaults to ``False`` 202 | :type update: bool 203 | :param download: whether to ask for confirmation 204 | before starting to download a file, defaults to ``True`` 205 | :type download: bool 206 | :param pickle_it: whether to save the .shp data as a pickle file, defaults to ``False`` 207 | :type pickle_it: bool 208 | :param ret_pickle_path: (when ``pickle_it=True``) 209 | whether to return a path to the saved pickle file 210 | :type ret_pickle_path: bool 211 | :param rm_extracts: whether to delete extracted files from the .shp.zip file, 212 | defaults to ``False`` 213 | :type rm_extracts: bool 214 | :param rm_shp_zip: whether to delete the downloaded .shp.zip file, defaults to ``False`` 215 | :type rm_shp_zip: bool 216 | :param verbose: whether to print relevant information in console as the function runs, 217 | defaults to ``False`` 218 | :type verbose: bool | int 219 | :return: dictionary of the shapefile data, with keys and values being layer names 220 | and tabular data (in the format of `geopandas.GeoDataFrame`_), respectively; 221 | when ``pickle_it=True``, return a tuple of the dictionary and a path to the pickle file 222 | :rtype: dict | collections.OrderedDict | tuple | None 223 | 224 | .. _`geopandas.GeoDataFrame`: https://geopandas.org/reference.html#geodataframe 225 | 226 | **Examples**:: 227 | 228 | >>> from pydriosm.reader import BBBikeReader 229 | >>> from pyhelpers.dirs import delete_dir 230 | >>> import os 231 | 232 | >>> bbr = BBBikeReader() 233 | 234 | >>> subrgn_name = 'Birmingham' 235 | >>> dat_dir = "tests\\osm_data" 236 | 237 | >>> bham_shp = bbr.read_shp_zip( 238 | ... subregion_name=subrgn_name, data_dir=dat_dir, download=False, verbose=True) 239 | The .shp.zip file for "Birmingham" is not found. 240 | 241 | >>> # Set `download=True` 242 | >>> bham_shp = bbr.read_shp_zip( 243 | ... subregion_name=subrgn_name, data_dir=dat_dir, download=True, verbose=True) 244 | Downloading "Birmingham.osm.shp.zip" 245 | to "tests\\osm_data\\birmingham\\" ... Done. 246 | Extracting "tests\\osm_data\\birmingham\\Birmingham.osm.shp.zip" 247 | to "tests\\osm_data\\birmingham\\" ... Done. 248 | Reading the shapefile(s) at 249 | "tests\\osm_data\\birmingham\\Birmingham-shp\\shape\\" ... Done. 250 | >>> type(bham_shp) 251 | collections.OrderedDict 252 | >>> list(bham_shp.keys()) 253 | ['buildings', 254 | 'landuse', 255 | 'natural', 256 | 'places', 257 | 'points', 258 | 'railways', 259 | 'roads', 260 | 'waterways'] 261 | 262 | >>> # Data of 'railways' layer 263 | >>> bham_railways_shp = bham_shp['railways'] 264 | >>> bham_railways_shp.head() 265 | osm_id ... shape_type 266 | 0 740 ... 3 267 | 1 2148 ... 3 268 | 2 2950000 ... 3 269 | 3 3491845 ... 3 270 | 4 3981454 ... 3 271 | [5 rows x 5 columns] 272 | 273 | >>> # Read data of 'road' layer only from the original .shp.zip file 274 | >>> # (and delete all extracts) 275 | >>> lyr_name = 'roads' 276 | >>> bham_roads_shp = bbr.read_shp_zip( 277 | ... subregion_name=subrgn_name, layer_names=lyr_name, data_dir=dat_dir, 278 | ... rm_extracts=True, verbose=True) 279 | Reading "tests\\osm_data\\birmingham\\Birmingham-shp\\shape\\roads.shp" ... Done. 280 | Deleting the extracts "tests\\osm_data\\birmingham\\Birmingham-shp\\" ... Done. 281 | >>> type(bham_roads_shp) 282 | collections.OrderedDict 283 | >>> list(bham_roads_shp.keys()) 284 | ['roads'] 285 | >>> bham_roads_shp[lyr_name].head() 286 | osm_id ... shape_type 287 | 0 37 ... 3 288 | 1 38 ... 3 289 | 2 41 ... 3 290 | 3 45 ... 3 291 | 4 46 ... 3 292 | [5 rows x 9 columns] 293 | 294 | >>> # Read data of multiple layers and features from the original .shp.zip file 295 | >>> # (and delete all extracts) 296 | >>> lyr_names = ['railways', 'waterways'] 297 | >>> feat_names = ['rail', 'canal'] 298 | >>> bham_rw_rc_shp = bbr.read_shp_zip( 299 | ... subregion_name=subrgn_name, layer_names=lyr_names, feature_names=feat_names, 300 | ... data_dir=dat_dir, rm_extracts=True, rm_shp_zip=True, verbose=True) 301 | Extracting the following layer(s): 302 | 'railways' 303 | 'waterways' 304 | from "tests\\osm_data\\birmingham\\Birmingham.osm.shp.zip" 305 | to "tests\\osm_data\\birmingham\\" ... Done. 306 | Reading the data at "tests\\osm_data\\birmingham\\Birmingham-shp\\shape\\" ... Done. 307 | Deleting the extracts "tests\\osm_data\\birmingham\\Birmingham-shp\\" ... Done. 308 | Deleting "tests\\osm_data\\birmingham\\Birmingham.osm.shp.zip" ... Done. 309 | >>> type(bham_rw_rc_shp) 310 | collections.OrderedDict 311 | >>> list(bham_rw_rc_shp.keys()) 312 | ['railways', 'waterways'] 313 | 314 | >>> # Data of the 'railways' layer 315 | >>> bham_rw_rc_shp_railways = bham_rw_rc_shp['railways'] 316 | >>> bham_rw_rc_shp_railways[['type', 'name']].head() 317 | type name 318 | 0 rail Cross-City Line 319 | 1 rail Cross-City Line 320 | 2 rail Derby to Birmingham (Proof House Junction) Line 321 | 3 rail Birmingham to Peterborough Line 322 | 4 rail Water Orton to Park Lane Junction Curve 323 | 324 | >>> # Data of the 'waterways' layer 325 | >>> bham_rw_rc_shp_waterways = bham_rw_rc_shp['waterways'] 326 | >>> bham_rw_rc_shp_waterways[['type', 'name']].head() 327 | type name 328 | 2 canal Birmingham and Fazeley Canal 329 | 8 canal Birmingham and Fazeley Canal 330 | 9 canal Birmingham Old Line Canal Navigations - Rotton P 331 | 10 canal Oozells Street Loop 332 | 11 canal Worcester & Birmingham Canal 333 | 334 | >>> # Delete the example data and the test data directory 335 | >>> delete_dir(dat_dir, verbose=True) 336 | To delete the directory "tests\\osm_data\\" (Not empty) 337 | ? [No]|Yes: yes 338 | Deleting "tests\\osm_data\\" ... Done. 339 | """ 340 | 341 | shp_data = super().read_shp_zip( 342 | subregion_name=subregion_name, layer_names=layer_names, feature_names=feature_names, 343 | data_dir=data_dir, update=update, download=download, pickle_it=pickle_it, 344 | ret_pickle_path=ret_pickle_path, rm_extracts=rm_extracts, rm_shp_zip=rm_shp_zip, 345 | verbose=verbose, **kwargs) 346 | 347 | return shp_data 348 | 349 | def read_csv_xz(self, subregion_name, data_dir=None, download=False, verbose=False, **kwargs): 350 | """ 351 | Read a compressed CSV (.csv.xz) data file of a geographic (sub)region. 352 | 353 | :param subregion_name: name of a geographic (sub)region (case-insensitive) 354 | that is available on BBBike free download server 355 | :type subregion_name: str 356 | :param data_dir: directory where the .csv.xz data file is located/saved; 357 | if ``None`` (default), the default directory 358 | :type data_dir: str | None 359 | :param download: whether to try to download the requisite data file if it does not exist, 360 | defaults to ``True`` 361 | :type download: bool 362 | :param verbose: whether to print relevant information in console as the function runs, 363 | defaults to ``False`` 364 | :type verbose: bool | int 365 | :return: tabular data of the .csv.xz file 366 | :rtype: pandas.DataFrame | None 367 | 368 | .. _pydriosm-BBBikeReader-read_csv_xz: 369 | 370 | **Examples**:: 371 | 372 | >>> from pydriosm.reader import BBBikeReader 373 | >>> from pyhelpers.dirs import cd, delete_dir 374 | 375 | >>> bbr = BBBikeReader() 376 | 377 | >>> subrgn_name = 'Leeds' 378 | >>> dat_dir = "tests\\osm_data" 379 | 380 | >>> leeds_csv_xz = bbr.read_csv_xz(subrgn_name, dat_dir, verbose=True) 381 | The requisite data file "tests\\osm_data\\leeds\\Leeds.osm.csv.xz" does not exist. 382 | 383 | >>> leeds_csv_xz = bbr.read_csv_xz(subrgn_name, dat_dir, verbose=True, download=True) 384 | Downloading "Leeds.osm.csv.xz" 385 | to "tests\\osm_data\\leeds\\" ... Done. 386 | Parsing the data ... Done. 387 | 388 | >>> leeds_csv_xz.head() 389 | type id feature note 390 | 0 node 154915 None None 391 | 1 node 154916 None None 392 | 2 node 154921 None None 393 | 3 node 154922 None None 394 | 4 node 154923 None None 395 | 396 | >>> # Delete the downloaded .csv.xz data file 397 | >>> delete_dir(dat_dir, verbose=True) 398 | To delete the directory "tests\\osm_data\\" (Not empty) 399 | ? [No]|Yes: yes 400 | Deleting "tests\\osm_data\\" ... Done. 401 | """ 402 | 403 | csv_xz_data = self.read_osm_var( 404 | self.VAR.read_csv_xz, subregion_name=subregion_name, osm_file_format=".csv.xz", 405 | data_dir=data_dir, download=download, verbose=verbose, **kwargs) 406 | 407 | return csv_xz_data 408 | 409 | def read_geojson_xz(self, subregion_name, data_dir=None, parse_geometry=False, download=False, 410 | verbose=False, **kwargs): 411 | """ 412 | Read a .geojson.xz data file of a geographic (sub)region. 413 | 414 | :param subregion_name: name of a geographic (sub)region (case-insensitive) 415 | that is available on BBBike free download server 416 | :type subregion_name: str 417 | :param data_dir: directory where the .geojson.xz data file is located/saved; 418 | if ``None`` (default), the default directory 419 | :type data_dir: str | None 420 | :param parse_geometry: whether to represent coordinates in a format of a geometric object, 421 | defaults to ``False`` 422 | :type parse_geometry: bool 423 | :param download: whether to try to download the requisite data file if it does not exist, 424 | defaults to ``True`` 425 | :type download: bool 426 | :param verbose: whether to print relevant information in console as the function runs, 427 | defaults to ``False`` 428 | :type verbose: bool | int 429 | :return: tabular data of the .csv.xz file 430 | :rtype: pandas.DataFrame | None 431 | 432 | .. _pydriosm-BBBikeReader-read_geojson_xz: 433 | 434 | **Examples**:: 435 | 436 | >>> from pydriosm.reader import BBBikeReader 437 | >>> from pyhelpers.dirs import cd, delete_dir 438 | >>> import os 439 | 440 | >>> bbr = BBBikeReader() 441 | 442 | >>> subrgn_name = 'Leeds' 443 | >>> dat_dir = "tests\\osm_data" 444 | 445 | >>> leeds_geoj = bbr.read_geojson_xz(subrgn_name, dat_dir, verbose=True) 446 | The requisite data file "tests\\osm_data\\leeds\\Leeds.osm.geojson.xz" does not exist. 447 | 448 | >>> # Set `try_download=True` 449 | >>> leeds_geoj = bbr.read_geojson_xz(subrgn_name, dat_dir, verbose=True, download=True) 450 | Downloading "Leeds.osm.geojson.xz" 451 | to "tests\\osm_data\\leeds\\" ... Done. 452 | Parsing the data ... Done. 453 | >>> leeds_geoj.head() 454 | geometry properties 455 | 0 {'type': 'Point', 'coordinates': [-1.5558097, ... {'highway': 'motorway_junction'... 456 | 1 {'type': 'Point', 'coordinates': [-1.34293, 53... {'highway': 'motorway_junction'... 457 | 2 {'type': 'Point', 'coordinates': [-1.517335, 5... {'highway': 'motorway_junction'... 458 | 3 {'type': 'Point', 'coordinates': [-1.514124, 5... {'highway': 'motorway_junction'... 459 | 4 {'type': 'Point', 'coordinates': [-1.516511, 5... {'highway': 'motorway_junction'... 460 | 461 | >>> # Set `parse_geometry` to be True 462 | >>> leeds_geoj_ = bbr.read_geojson_xz(subrgn_name, dat_dir, parse_geometry=True, 463 | ... verbose=True) 464 | Parsing "tests\\osm_data\\leeds\\Leeds.osm.geojson.xz" ... Done. 465 | >>> leeds_geoj_['geometry'].head() 466 | 0 POINT (-1.5560511 53.6879848) 467 | 1 POINT (-1.34293 53.844618) 468 | 2 POINT (-1.517335 53.7499667) 469 | 3 POINT (-1.514124 53.7416937) 470 | 4 POINT (-1.516511 53.7256632) 471 | Name: geometry, dtype: object 472 | 473 | >>> # Delete the download directory 474 | >>> delete_dir(dat_dir, verbose=True) 475 | """ 476 | 477 | kwargs.update({'parse_geometry': parse_geometry}) 478 | 479 | geojson_xz_data = self.read_osm_var( 480 | self.VAR.read_geojson_xz, subregion_name=subregion_name, osm_file_format=".geojson.xz", 481 | data_dir=data_dir, download=download, verbose=verbose, **kwargs) 482 | 483 | return geojson_xz_data 484 | -------------------------------------------------------------------------------- /pydriosm/reader/transformer.py: -------------------------------------------------------------------------------- 1 | """ 2 | Transform the OSM data. 3 | """ 4 | 5 | import copy 6 | import re 7 | 8 | import pandas as pd 9 | import shapely.errors 10 | import shapely.geometry 11 | from pyhelpers.ops import update_dict 12 | 13 | from pydriosm.errors import OtherTagsReformatError 14 | 15 | 16 | class Transformer: 17 | """ 18 | Transform / reformat data. 19 | 20 | **Examples**:: 21 | 22 | >>> from pydriosm.reader import Transformer 23 | 24 | >>> geometry = {'type': 'Point', 'coordinates': [-0.5134241, 52.6555853]} 25 | >>> geometry_ = Transformer.transform_unitary_geometry(geometry) 26 | >>> type(geometry_) 27 | shapely.geometry.point.Point 28 | >>> geometry_.wkt 29 | 'POINT (-0.5134241 52.6555853)' 30 | """ 31 | 32 | @classmethod 33 | def point_as_polygon(cls, multi_poly_coords): 34 | """ 35 | Make the coordinates of a single 'Point' (in a 'MultiPolygon') be reformatted to 36 | a 'Polygon'-like coordinates. 37 | 38 | The list of coordinates of some 'MultiPolygon' features may contain single points. 39 | In order to reformat such multipart geometry (from dict into `shapely.geometry`_ type), 40 | there is a need to ensure each of the constituent parts is a `shapely.geometry.Polygon`_. 41 | 42 | :param multi_poly_coords: original data of coordinates of a 43 | `shapely.geometry.MultiPolygon`_ feature 44 | :type multi_poly_coords: list 45 | :return: coordinates that are reformatted as appropriate 46 | :rtype: list 47 | 48 | .. _`shapely.geometry`: 49 | https://shapely.readthedocs.io/en/latest/manual.html#geometric-objects 50 | .. _`shapely.geometry.Polygon`: 51 | https://shapely.readthedocs.io/en/stable/manual.html#Polygon 52 | .. _`shapely.geometry.MultiPolygon`: 53 | https://shapely.readthedocs.io/en/stable/manual.html#MultiPolygon 54 | 55 | **Examples**:: 56 | 57 | >>> from pydriosm.reader import Transformer 58 | 59 | >>> geometry = { 60 | ... 'type': 'MultiPolygon', 61 | ... 'coordinates': [[[[-0.6920145, 52.6753268], [-0.6920145, 52.6753268]]]] 62 | ... } 63 | >>> mp_coords = geometry['coordinates'] 64 | 65 | >>> mp_coords_ = Transformer.point_as_polygon(mp_coords) 66 | >>> mp_coords_ 67 | [[[[-0.6920145, 52.6753268], 68 | [-0.6920145, 52.6753268], 69 | [-0.6920145, 52.6753268]]]] 70 | """ 71 | 72 | coords = multi_poly_coords.copy() 73 | temp = coords[0][0] 74 | 75 | if len(temp) == 2 and temp[0] == temp[1]: 76 | coords[0][0] += [temp[0]] 77 | 78 | return coords 79 | 80 | @classmethod 81 | def transform_unitary_geometry(cls, geometry, mode=1, to_wkt=False): 82 | """ 83 | Transform a unitary geometry from dict into a `shapely.geometry`_ object. 84 | 85 | :param geometry: geometry data for a feature of one of the geometry types including 86 | ``'Point'``, ``'LineString'``, ``'MultiLineString'`` and ``'MultiPolygon'`` 87 | :type geometry: dict | pandas.DataFrame 88 | :param mode: indicate the way of parsing the input; 89 | 90 | - when ``mode=1`` **(default)**, the input ``geometry`` should be directly accessible 91 | and would be in the format of ``{'type': , 'coordinates': }`` 92 | or as a row of a `pandas.DataFrame`_; 93 | - when ``mode=2``, the input ``geometry`` is in the `GeoJSON`_ format 94 | 95 | :type mode: int 96 | :param to_wkt: whether to represent the geometry in the WKT (well-known text) format, 97 | defaults to ``False`` 98 | :type to_wkt: bool 99 | :return: reformatted geometry data 100 | :rtype: shapely.geometry.Point | dict | str 101 | 102 | .. _`shapely.geometry`: 103 | https://shapely.readthedocs.io/en/latest/manual.html#geometric-objects 104 | .. _`pandas.DataFrame`: 105 | https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.html 106 | .. _`GeoJSON`: 107 | https://geojson.org/ 108 | 109 | **Examples**:: 110 | 111 | >>> from pydriosm.reader import PBFReadParse 112 | 113 | >>> g1_dat = {'type': 'Point', 'coordinates': [-0.5134241, 52.6555853]} 114 | >>> g1_data = PBFReadParse.transform_unitary_geometry(g1_dat) 115 | >>> type(g1_data) 116 | shapely.geometry.point.Point 117 | >>> g1_data.wkt 118 | 'POINT (-0.5134241 52.6555853)' 119 | 120 | >>> g2_dat = { 121 | ... 'type': 'Feature', 122 | ... 'geometry': { 123 | ... 'type': 'Point', 124 | ... 'coordinates': [-0.5134241, 52.6555853] 125 | ... }, 126 | ... 'properties': { 127 | ... 'osm_id': '488432', 128 | ... 'name': None, 129 | ... 'barrier': None, 130 | ... 'highway': None, 131 | ... 'ref': None, 132 | ... 'address': None, 133 | ... 'is_in': None, 134 | ... 'place': None, 135 | ... 'man_made': None, 136 | ... 'other_tags': '"odbl"=>"clean"' 137 | ... }, 138 | ... 'id': 488432 139 | ... } 140 | >>> g2_data = PBFReadParse.transform_unitary_geometry(g2_dat, mode=2) 141 | >>> type(g2_data) 142 | dict 143 | >>> list(g2_data.keys()) 144 | ['type', 'geometry', 'properties', 'id'] 145 | >>> g2_data['geometry'] 146 | 'POINT (-0.5134241 52.6555853)' 147 | """ 148 | 149 | if mode == 1: 150 | geom_type, coords = geometry['type'], geometry['coordinates'] 151 | geom_func = getattr(shapely.geometry, geom_type) 152 | 153 | if geom_type == 'MultiPolygon': 154 | geom_data = geom_func( 155 | shapely.geometry.Polygon(y) for x in cls.point_as_polygon(coords) for y in x) 156 | # geom_data = geom.wkt if to_wkt else geom.geoms 157 | 158 | else: 159 | geom_data = geom_func(coords) 160 | # if to_wkt: 161 | # geom_data = geom_data.wkt 162 | # elif 'Multi' in geom_type: 163 | # geom_data = geom_data.geoms 164 | 165 | if to_wkt: 166 | geom_data = geom_data.wkt 167 | 168 | else: 169 | geom_data = geometry.copy() 170 | dat = cls.transform_unitary_geometry(geometry['geometry'], mode=1, to_wkt=True) 171 | geom_data.update({'geometry': dat}) 172 | 173 | return geom_data 174 | 175 | @classmethod 176 | def transform_geometry_collection(cls, geometry, mode=1, to_wkt=False): 177 | """ 178 | Transform a collection of geometry from dict into a `shapely.geometry`_ object. 179 | 180 | :param geometry: geometry data for a feature of ``GeometryCollection`` 181 | :type geometry: list | dict 182 | :param mode: indicate the way of parsing the input; 183 | 184 | - when ``mode=1`` **(default)**, the input ``geometry`` should be directly accessible 185 | and would be in the format of ``{'type': , 'coordinates': }`` 186 | or as a row of a `pandas.DataFrame`_; 187 | - when ``mode=2``, the input ``geometry`` is in the `GeoJSON`_ format 188 | 189 | :type mode: int 190 | :param to_wkt: whether to represent the geometry in the WKT (well-known text) format, 191 | defaults to ``False`` 192 | :type to_wkt: bool 193 | :return: reformatted geometry data 194 | :rtype: shapely.geometry.base.HeterogeneousGeometrySequence | dict | str 195 | 196 | .. _`shapely.geometry`: 197 | https://shapely.readthedocs.io/en/latest/manual.html#geometric-objects 198 | .. _`pandas.DataFrame`: 199 | https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.html 200 | .. _`GeoJSON`: 201 | https://geojson.org/ 202 | 203 | **Examples**:: 204 | 205 | >>> from pydriosm.reader import PBFReadParse 206 | >>> from shapely.geometry import GeometryCollection 207 | 208 | >>> g1_dat_ = { 209 | ... 'type': 'GeometryCollection', 210 | ... 'geometries': [ 211 | ... {'type': 'Point', 'coordinates': [-0.5096176, 52.6605168]}, 212 | ... {'type': 'Point', 'coordinates': [-0.5097337, 52.6605812]} 213 | ... ] 214 | ... } 215 | >>> g1_dat = g1_dat_['geometries'] 216 | >>> g1_data = PBFReadParse.transform_geometry_collection(g1_dat) 217 | >>> type(g1_data) 218 | shapely.geometry.collection.GeometryCollection 219 | >>> g1_data.wkt 220 | 'GEOMETRYCOLLECTION (POINT (-0.5096176 52.6605168), POINT (-0.5097337 52.6605812))' 221 | 222 | >>> g2_dat = { 223 | ... 'type': 'Feature', 224 | ... 'geometry': { 225 | ... 'type': 'GeometryCollection', 226 | ... 'geometries': [ 227 | ... {'type': 'Point', 'coordinates': [-0.5096176, 52.6605168]}, 228 | ... {'type': 'Point', 'coordinates': [-0.5097337, 52.6605812]}] 229 | ... }, 230 | ... 'properties': { 231 | ... 'osm_id': '256254', 232 | ... 'name': 'Fife Close', 233 | ... 'type': 'site', 234 | ... 'other_tags': '"naptan:StopAreaCode"=>"270G02701525"' 235 | ... }, 236 | ... 'id': 256254 237 | ... } 238 | >>> g2_data = PBFReadParse.transform_geometry_collection(g2_dat, mode=2) 239 | >>> type(g2_data) 240 | dict 241 | >>> list(g2_data.keys()) 242 | ['type', 'geometry', 'properties', 'id'] 243 | >>> g2_data['geometry'] 244 | 'GEOMETRYCOLLECTION (POINT (-0.5096176 52.6605168), POINT (-0.5097337 52.6605812))' 245 | """ 246 | 247 | if mode == 1: 248 | geometry_collection = [] 249 | 250 | for geom_type, coords in zip(*zip(*map(dict.values, geometry))): 251 | geom_func = getattr(shapely.geometry, geom_type) 252 | if 'Polygon' not in geom_type: 253 | geometry_collection.append(geom_func(coords)) 254 | else: 255 | geometry_collection.append(geom_func(pt for pts in coords for pt in pts)) 256 | 257 | geome_data = shapely.geometry.GeometryCollection(geometry_collection) 258 | if to_wkt: 259 | geome_data = geome_data.wkt 260 | # else: 261 | # geome_data = shapely.geometry.GeometryCollection(geometry_collection).geoms 262 | 263 | else: 264 | geome_data = geometry.copy() 265 | geometries = geome_data['geometry']['geometries'] 266 | geome_data.update( 267 | {'geometry': cls.transform_geometry_collection(geometries, mode=1, to_wkt=True)}) 268 | 269 | return geome_data 270 | 271 | @classmethod 272 | def transform_geometry(cls, layer_data, layer_name): 273 | """ 274 | Reformat the field of ``'geometry'`` into 275 | `shapely.geometry `_ 276 | object. 277 | 278 | :param layer_data: dataframe of a specific layer of PBF data 279 | :type layer_data: pandas.DataFrame | pandas.Series 280 | :param layer_name: name (geometric type) of the PBF layer 281 | :type layer_name: str 282 | :return: (OSM feature with) reformatted geometry field 283 | :rtype: pandas.DataFrame | pandas.Series 284 | 285 | **Examples**:: 286 | 287 | >>> from pydriosm.reader import Transformer 288 | 289 | >>> # An example of points layer data 290 | >>> lyr_name = 'points' 291 | >>> dat_ = { 292 | ... 'type': 'Feature', 293 | ... 'geometry': { 294 | ... 'type': 'Point', 295 | ... 'coordinates': [-0.5134241, 52.6555853] 296 | ... }, 297 | ... 'properties': { 298 | ... 'osm_id': '488432', 299 | ... 'name': None, 300 | ... 'barrier': None, 301 | ... 'highway': None, 302 | ... 'ref': None, 303 | ... 'address': None, 304 | ... 'is_in': None, 305 | ... 'place': None, 306 | ... 'man_made': None, 307 | ... 'other_tags': '"odbl"=>"clean"' 308 | ... }, 309 | ... 'id': 488432 310 | ... } 311 | >>> lyr_data = pd.DataFrame.from_dict(dat_, orient='index').T 312 | 313 | >>> geom_dat = Transformer.transform_geometry(layer_data=lyr_data, layer_name=lyr_name) 314 | >>> geom_dat 315 | 0 POINT (-0.5134241 52.6555853) 316 | Name: geometry, dtype: object 317 | 318 | .. seealso:: 319 | 320 | - Examples for the method 321 | :meth:`PBFReadParse.read_pbf()`. 322 | """ 323 | 324 | geom_col_name = 'geometry' 325 | 326 | if layer_name == 'other_relations': 327 | if isinstance(layer_data, pd.DataFrame): # geom_col_name in layer_data.columns: 328 | geometries = pd.DataFrame(list(layer_data[geom_col_name]))['geometries'] 329 | geom_data = geometries.map(cls.transform_geometry_collection) 330 | geom_data.name = geom_col_name 331 | else: 332 | geom_data = layer_data.map(lambda x: cls.transform_geometry_collection(x, mode=2)) 333 | 334 | else: # `layer_data` can be 'points', 'lines', 'multilinestrings' or 'multipolygons' 335 | if isinstance(layer_data, pd.DataFrame): # geom_col_name in layer_data.columns: 336 | geom_data = layer_data[geom_col_name].map(cls.transform_unitary_geometry) 337 | else: 338 | geom_data = layer_data.map(lambda x: cls.transform_unitary_geometry(x, mode=2)) 339 | 340 | return geom_data 341 | 342 | @classmethod 343 | def transform_other_tags(cls, other_tags): 344 | """ 345 | Reformat a ``'other_tags'`` from string into dictionary type. 346 | 347 | :param other_tags: data of ``'other_tags'`` of a single feature in a PBF data file 348 | :type other_tags: str | None 349 | :return: reformatted data of ``'other_tags'`` 350 | :rtype: dict | None 351 | 352 | **Examples**:: 353 | 354 | >>> from pydriosm.reader import Transformer 355 | 356 | >>> other_tags_dat = Transformer.transform_other_tags(other_tags='"odbl"=>"clean"') 357 | >>> other_tags_dat 358 | {'odbl': 'clean'} 359 | 360 | .. seealso:: 361 | 362 | - Examples for the method 363 | :meth:`PBFReadParse.read_pbf()`. 364 | """ 365 | 366 | if other_tags: 367 | tags = [re.sub(r'^"|"$', '', x) for x in re.split('(?<="),(?=")', other_tags)] 368 | try: 369 | fltr = (re.split(r'"=>"?', x, maxsplit=1) for x in filter(None, tags)) 370 | except OtherTagsReformatError: 371 | fltr = filter( 372 | lambda x: len(x) == 2, (re.split(r'"=>"?', x) for x in filter(None, tags))) 373 | other_tags_ = {k: v.replace('
', ' ') for k, v in fltr} 374 | 375 | else: # e.g. the data of 'other_tags' is None 376 | other_tags_ = other_tags 377 | 378 | return other_tags_ 379 | 380 | @classmethod 381 | def update_other_tags(cls, prop_or_feat, mode=1): 382 | """ 383 | Update the original data of ``'other_tags'`` with parsed data. 384 | 385 | :param prop_or_feat: original data of a feature or a ``'properties'`` field 386 | :type prop_or_feat: dict 387 | :param mode: options include ``{1, 2}`` indicating what action to take; 388 | when ``mode=1`` (default), ``prop_or_feat`` should be data of a feature; 389 | when ``mode=2``, ``prop_or_feat`` should be data of a ``'properties'`` field 390 | :type mode: int 391 | :return: updated data of a feature or a 'properties' field 392 | :rtype: dict 393 | 394 | **Examples**:: 395 | 396 | >>> from pydriosm.reader import Transformer 397 | 398 | >>> prop_dat = { 399 | ... 'properties': { 400 | ... 'osm_id': '488432', 401 | ... 'name': None, 402 | ... 'barrier': None, 403 | ... 'highway': None, 404 | ... 'ref': None, 405 | ... 'address': None, 406 | ... 'is_in': None, 407 | ... 'place': None, 408 | ... 'man_made': None, 409 | ... 'other_tags': '"odbl"=>"clean"' 410 | ... }, 411 | ... } 412 | 413 | >>> prop_dat_ = Transformer.update_other_tags(prop_dat['properties']) 414 | >>> prop_dat_ 415 | {'osm_id': '488432', 416 | 'name': None, 417 | 'barrier': None, 418 | 'highway': None, 419 | 'ref': None, 420 | 'address': None, 421 | 'is_in': None, 422 | 'place': None, 423 | 'man_made': None, 424 | 'other_tags': {'odbl': 'clean'}} 425 | 426 | .. seealso:: 427 | 428 | - Examples for the method 429 | :meth:`PBFReadParse.read_pbf()`. 430 | """ 431 | 432 | if mode == 1: 433 | properties = prop_or_feat.copy() 434 | properties.update({'other_tags': cls.transform_other_tags(properties['other_tags'])}) 435 | 436 | return properties 437 | 438 | else: 439 | feat = prop_or_feat.copy() 440 | other_tags = copy.copy(feat['properties']['other_tags']) 441 | 442 | feat_ = update_dict( 443 | feat, {'properties': {'other_tags': cls.transform_other_tags(other_tags)}}) 444 | # feat['properties'].update({'other_tags': transform_other_tags(other_tags)}) 445 | 446 | return feat_ 447 | -------------------------------------------------------------------------------- /pydriosm/utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | Provide various helper functions for use across the package. 3 | """ 4 | 5 | import importlib.resources 6 | import os 7 | import shutil 8 | 9 | from pyhelpers._cache import _check_dependency, _format_err_msg 10 | from pyhelpers.dirs import cd 11 | 12 | 13 | # ================================================================================================== 14 | # Data directories 15 | # ================================================================================================== 16 | 17 | 18 | def _cdd(*sub_dir, data_dir="data", mkdir=False, **kwargs): 19 | """ 20 | Specify (or change to) a directory (or any subdirectories) for backup data of the package. 21 | 22 | :param sub_dir: [optional] name of a directory; names of directories (and/or a filename) 23 | :type sub_dir: str | os.PathLike[str] 24 | :param data_dir: name of a directory to store data, defaults to ``"data"`` 25 | :type data_dir: str | os.PathLike[str] 26 | :param mkdir: whether to create a directory, defaults to ``False`` 27 | :type mkdir: bool 28 | :param kwargs: [optional] parameters (e.g. ``mode=0o777``) of `os.makedirs`_ 29 | :return: a full pathname of a directory or a file under the specified data directory ``data_dir`` 30 | :rtype: str 31 | 32 | .. _`os.makedirs`: https://docs.python.org/3/library/os.html#os.makedirs 33 | 34 | **Example**:: 35 | 36 | >>> from pydriosm.utils import _cdd 37 | >>> import os 38 | 39 | >>> path_to_dat_dir = _cdd(data_dir="data") 40 | >>> os.path.relpath(path_to_dat_dir) 41 | 'pydriosm\\data' 42 | """ 43 | 44 | pathname = importlib.resources.files(__package__).joinpath(data_dir) 45 | for x in sub_dir: 46 | pathname = os.path.join(pathname, x) 47 | 48 | if mkdir: 49 | path_to_file, ext = os.path.splitext(pathname) 50 | if ext == '': 51 | os.makedirs(path_to_file, exist_ok=True, **kwargs) 52 | else: 53 | os.makedirs(os.path.dirname(pathname), exist_ok=True, **kwargs) 54 | 55 | return pathname 56 | 57 | 58 | def check_relpath(pathname, start=os.curdir): 59 | """ 60 | Check and return a relative pathname to the given ``pathname``. 61 | 62 | On Windows, when ``pathname`` and ``start`` are on different drives, the function returns 63 | the given ``pathname``. 64 | 65 | :param pathname: pathname of a file or a directory 66 | :type pathname: str | os.PathLike[str] 67 | :param start: optional start directory, 68 | defaults to ``os.curdir`` (i.e. the current working directory) 69 | :type start: str | os.PathLike[str] 70 | :return: relative pathname to the given ``pathname`` 71 | :type: str | os.PathLike[str] 72 | """ 73 | 74 | try: 75 | relpath = os.path.relpath(pathname, start=start) 76 | except ValueError: 77 | relpath = pathname 78 | 79 | return relpath 80 | 81 | 82 | def cdd_geofabrik(*sub_dir, mkdir=False, default_dir="osm_geofabrik", **kwargs): 83 | """ 84 | Change directory to ``osm_geofabrik\\`` and its subdirectories within a package. 85 | 86 | :param sub_dir: name of directory; names of directories (and/or a filename) 87 | :type sub_dir: str | os.PathLike 88 | :param mkdir: whether to create a directory, defaults to ``False`` 89 | :type mkdir: bool 90 | :param default_dir: default folder name of the root directory for downloading data from Geofabrik, 91 | defaults to ``"osm_geofabrik"`` 92 | :type default_dir: str 93 | :param kwargs: [optional] parameters of `pyhelpers.dir.cd()`_ 94 | :return: an absolute path to a directory (or a file) under ``data_dir`` 95 | :rtype: str | os.PathLike 96 | 97 | .. _`pyhelpers.dir.cd()`: 98 | https://pyhelpers.readthedocs.io/en/latest/_generated/pyhelpers.dir.cd.html 99 | 100 | **Examples**:: 101 | 102 | >>> from pydriosm.utils import cdd_geofabrik 103 | >>> import os 104 | 105 | >>> os.path.relpath(cdd_geofabrik()) 106 | 'osm_geofabrik' 107 | """ 108 | 109 | pathname = cd(default_dir, *sub_dir, mkdir=mkdir, **kwargs) 110 | 111 | return pathname 112 | 113 | 114 | def cdd_bbbike(*sub_dir, mkdir=False, default_dir="osm_bbbike", **kwargs): 115 | """ 116 | Change directory to ``osm_bbbike\\`` and its subdirectories. 117 | 118 | :param sub_dir: name of directory; names of directories (and/or a filename) 119 | :type sub_dir: str 120 | :param mkdir: whether to create a directory, defaults to ``False`` 121 | :type mkdir: bool 122 | :param default_dir: default folder name of the root directory for downloading data from BBBike, 123 | defaults to ``"osm_bbbike"`` 124 | :type default_dir: str 125 | :param kwargs: [optional] parameters of `pyhelpers.dir.cd()`_ 126 | :return: an absolute path to a directory (or a file) under ``data_dir`` 127 | :rtype: str 128 | 129 | .. _`pyhelpers.dir.cd()`: 130 | https://pyhelpers.readthedocs.io/en/latest/_generated/pyhelpers.dir.cd.html 131 | 132 | **Examples**:: 133 | 134 | >>> from pydriosm.utils import cdd_bbbike 135 | >>> import os 136 | 137 | >>> os.path.relpath(cdd_bbbike()) 138 | 'osm_bbbike' 139 | """ 140 | 141 | pathname = cd(default_dir, *sub_dir, mkdir=mkdir, **kwargs) 142 | 143 | return pathname 144 | 145 | 146 | # ================================================================================================== 147 | # General utilities 148 | # ================================================================================================== 149 | 150 | 151 | def first_unique(iterable): 152 | """ 153 | Return unique items in an input iterable variable given the same order of presence. 154 | 155 | :param iterable: iterable variable 156 | :type iterable: typing.Iterable 157 | :return: unique items in the same order of presence as in the input 158 | :rtype: typing.Generator[list] 159 | 160 | **Examples**:: 161 | 162 | >>> from pydriosm.utils import first_unique 163 | 164 | >>> list_example1 = [1, 2, 2, 3, 4, 5, 6, 6, 2, 3, 1, 6] 165 | >>> list(first_unique(list_example1)) 166 | [1, 2, 3, 4, 5, 6] 167 | 168 | >>> list_example2 = [6, 1, 2, 2, 3, 4, 5, 6, 6, 2, 3, 1] 169 | >>> list(first_unique(list_example2)) 170 | [6, 1, 2, 3, 4, 5] 171 | """ 172 | 173 | checked_list = [] 174 | 175 | for x in iterable: 176 | if x not in checked_list: 177 | checked_list.append(x) 178 | yield x 179 | 180 | 181 | def check_json_engine(engine=None): 182 | """ 183 | Check an available module used for loading JSON data. 184 | 185 | :param engine: name of a module for loading JSON data; 186 | when ``engine=None`` (default), use the built-in 187 | `json `_ module; 188 | :type engine: str | None 189 | :return: the module for loading JSON data 190 | :type: types.ModuleType | None 191 | 192 | **Examples**:: 193 | 194 | >>> from pydriosm.utils import check_json_engine 195 | >>> import types 196 | 197 | >>> result = check_json_engine() 198 | 199 | >>> isinstance(result, types.ModuleType) 200 | True 201 | >>> result.__name__ == 'json' 202 | True 203 | """ 204 | 205 | if engine is not None: 206 | valid_mod_names = {'ujson', 'orjson', 'rapidjson', 'json'} 207 | assert engine in valid_mod_names, f"`engine` must be on one of {valid_mod_names}." 208 | engine_ = _check_dependency(name=engine) 209 | 210 | else: 211 | engine_ = _check_dependency(name='json') 212 | 213 | return engine_ 214 | 215 | 216 | def remove_osm_file(path_to_file, verbose=True): 217 | """ 218 | Remove a downloaded OSM data file. 219 | 220 | :param path_to_file: absolute path to a downloaded OSM data file 221 | :type path_to_file: str 222 | :param verbose: defaults to ``True`` 223 | :type verbose: bool 224 | 225 | **Examples**:: 226 | 227 | >>> from pydriosm.utils import remove_osm_file 228 | >>> from pyhelpers.dirs import cd 229 | >>> import os 230 | 231 | >>> path_to_pseudo_pbf_file = cd('tests\\pseudo.osm.pbf') 232 | >>> try: 233 | ... open(path_to_pseudo_pbf_file, 'a').close() 234 | ... except OSError: 235 | ... print('Failed to create the file.') 236 | ... else: 237 | ... print('File created successfully.') 238 | File created successfully. 239 | 240 | >>> os.path.exists(path_to_pseudo_pbf_file) 241 | True 242 | 243 | >>> remove_osm_file(path_to_pseudo_pbf_file, verbose=True) 244 | Deleting "tests\\pseudo.osm.pbf" ... Done. 245 | 246 | >>> os.path.exists(path_to_pseudo_pbf_file) 247 | False 248 | """ 249 | 250 | if not os.path.exists(path_to_file): 251 | if verbose: 252 | print("The file \"{}\" is not found at {}.".format(*os.path.split(path_to_file)[::-1])) 253 | 254 | else: 255 | if verbose: 256 | print(f"Deleting \"{check_relpath(path_to_file)}\"", end=" ... ") 257 | 258 | try: 259 | if os.path.isfile(path_to_file): 260 | os.remove(path_to_file) 261 | if verbose: 262 | print("Done.") 263 | 264 | elif os.path.isdir(path_to_file): 265 | shutil.rmtree(path_to_file) 266 | if verbose: 267 | print("Done.") 268 | 269 | except Exception as e: 270 | print(f"Failed. {_format_err_msg(e)}") 271 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = [ 3 | "setuptools>=67.0", 4 | "wheel" 5 | ] 6 | build-backend = "setuptools.build_meta" -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | build==1.0.3 2 | fqdn==1.5.1 3 | gdal==3.4.3 4 | geopandas==0.14.0 5 | isoduration==20.11.0 6 | jsonpointer==2.4 7 | notebook==7.0.3 8 | orjson==3.9.7 9 | pip-chill==1.0.3 10 | pyrcs==0.3.7 11 | pyshp==2.3.1 12 | pytest-cov==4.1.0 13 | python-rapidjson==1.11 14 | sphinx-copybutton==0.5.2 15 | sphinx-rtd-theme==1.3.0 16 | tqdm==4.66.1 17 | twine==4.0.2 18 | ujson==5.8.0 19 | uri-template==1.3.0 20 | webcolors==1.13 21 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | long_description = file: README.md 3 | long_description_content_type = text/markdown 4 | keywords = 5 | Python 6 | OpenStreetMap 7 | OSM 8 | Geofabrik 9 | BBBike 10 | Protocolbuffer Binary Format 11 | PBF 12 | Shapefile 13 | PostgreSQL 14 | platforms = nt, posix 15 | classifiers = 16 | Intended Audience :: Developers 17 | Intended Audience :: Education 18 | Intended Audience :: End Users/Desktop 19 | Intended Audience :: Information Technology 20 | Intended Audience :: Science/Research 21 | Topic :: Education 22 | Topic :: Scientific/Engineering 23 | Topic :: Software Development 24 | Topic :: Utilities 25 | License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+) 26 | Programming Language :: Python :: 3 27 | Operating System :: Microsoft :: Windows 28 | Operating System :: POSIX :: Linux 29 | license_files = LICENSE 30 | 31 | [options] 32 | packages = 33 | pydriosm 34 | pydriosm.data 35 | include_package_data = True 36 | python_requires = >=3.9 37 | install_requires = 38 | pyhelpers>=1.5.2 39 | pyrcs>=0.3.7 40 | pyshp>=2.3.1 41 | 42 | [options.package_data] 43 | * = data/* 44 | 45 | [options.packages.find] 46 | exclude = 47 | tests 48 | *.tests 49 | tests.* 50 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | import setuptools 4 | 5 | with open(file="pydriosm/data/metadata.json", mode='r') as metadata_file: 6 | metadata = json.load(metadata_file) 7 | 8 | __pkgname__, __version__ = metadata['Package'], metadata['Version'] 9 | 10 | __home_page__ = 'https://github.com/mikeqfu/' + f'{__pkgname__}' 11 | 12 | setuptools.setup( 13 | name=__pkgname__, 14 | version=__version__, 15 | description=metadata['Description'], 16 | url=__home_page__, 17 | author=metadata['Author'], 18 | author_email=metadata['Email'], 19 | license=metadata['License'], 20 | project_urls={ 21 | 'Documentation': f'https://{__pkgname__}.readthedocs.io/en/{__version__}/', 22 | 'Source': __home_page__, 23 | 'Issue Tracker': __home_page__ + '/issues', 24 | }, 25 | ) 26 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/tests/__init__.py -------------------------------------------------------------------------------- /tests/data/rutland/other_relations_1.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/tests/data/rutland/other_relations_1.pkl -------------------------------------------------------------------------------- /tests/data/rutland/other_relations_2.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/tests/data/rutland/other_relations_2.pkl -------------------------------------------------------------------------------- /tests/data/rutland/points_1.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/tests/data/rutland/points_1.pkl -------------------------------------------------------------------------------- /tests/data/rutland/points_2.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/tests/data/rutland/points_2.pkl -------------------------------------------------------------------------------- /tests/data/rutland/rutland-latest-free.shp.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/tests/data/rutland/rutland-latest-free.shp.zip -------------------------------------------------------------------------------- /tests/data/rutland/rutland-latest.osm.pbf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/tests/data/rutland/rutland-latest.osm.pbf -------------------------------------------------------------------------------- /tests/test_downlaoder.py: -------------------------------------------------------------------------------- 1 | """Test the module :py:mod:`pydriosm.downloader`.""" 2 | 3 | import os 4 | import tempfile 5 | 6 | import pandas as pd 7 | import pytest 8 | from pyhelpers.dirs import delete_dir 9 | 10 | from pydriosm.downloader import BBBikeDownloader, GeofabrikDownloader 11 | from pydriosm.downloader._downloader import _Downloader 12 | from pydriosm.errors import InvalidFileFormatError, InvalidSubregionNameError 13 | 14 | gfd, bbd = GeofabrikDownloader(), BBBikeDownloader() 15 | 16 | 17 | class TestDownloader: 18 | 19 | @staticmethod 20 | def test_init(): 21 | d = _Downloader() 22 | 23 | assert d.NAME == 'OSM Downloader' 24 | assert os.path.relpath(d.download_dir) == 'osm_data' 25 | assert os.path.relpath(d.cdd()) == 'osm_data' 26 | assert d.download_dir == d.cdd() 27 | 28 | d = _Downloader(download_dir="tests\\osm_data") 29 | assert os.path.relpath(d.download_dir) == 'tests\\osm_data' 30 | 31 | @staticmethod 32 | def test_cdd(): 33 | assert os.path.relpath(_Downloader.cdd()) == 'osm_data' 34 | 35 | @staticmethod 36 | def test_compose_cfm_msg(): 37 | assert _Downloader.compose_cfm_msg() == 'To compile data of \n?' 38 | assert _Downloader.compose_cfm_msg(update=True) == 'To update the data of \n?' 39 | 40 | @staticmethod 41 | def test_print_act_msg(capfd): 42 | assert _Downloader.print_act_msg(verbose=False) is None 43 | 44 | _Downloader.print_act_msg(verbose=True) 45 | print("Done.") 46 | out, _ = capfd.readouterr() 47 | assert out == 'Compiling the data ... Done.\n' 48 | 49 | _Downloader.print_act_msg(verbose=True, note="(Some notes here.)") 50 | print("Done.") 51 | out, _ = capfd.readouterr() 52 | assert out == 'Compiling the data (Some notes here.) ... Done.\n' 53 | 54 | _Downloader.print_act_msg(verbose=True, confirmation_required=False) 55 | print("Done.") 56 | out, _ = capfd.readouterr() 57 | assert out == 'Compiling data of ... Done.\n' 58 | 59 | @staticmethod 60 | def test_print_otw_msg(capfd): 61 | assert _Downloader.print_otw_msg() is None 62 | 63 | _Downloader.print_otw_msg(verbose=True) 64 | out, _ = capfd.readouterr() 65 | assert out == 'Cancelled.\n' 66 | 67 | _Downloader.print_otw_msg(verbose=2) 68 | out, _ = capfd.readouterr() 69 | assert out == 'The collecting of is cancelled, or no data is available.\n' 70 | 71 | _Downloader.print_otw_msg(verbose=True, error_message="Errors.") 72 | out, _ = capfd.readouterr() 73 | assert out == 'Failed. Errors.\n' 74 | 75 | @staticmethod 76 | @pytest.mark.parametrize('data_name', [None, '']) 77 | def test_get_prepacked_data(data_name, monkeypatch, capfd): 78 | rslt = _Downloader.get_prepacked_data( 79 | callable, data_name=data_name, confirmation_required=False) 80 | assert rslt is None 81 | 82 | monkeypatch.setattr('builtins.input', lambda _: "No") 83 | rslt = _Downloader.get_prepacked_data(callable, verbose=True) 84 | out, _ = capfd.readouterr() 85 | assert 'Cancelled.' in out 86 | assert rslt is None 87 | 88 | @staticmethod 89 | def test_validate_subregion_name(): 90 | with pytest.raises(InvalidSubregionNameError) as e: 91 | _Downloader.validate_subregion_name('abc') 92 | assert '1)' in e and '2)' in e 93 | 94 | with pytest.raises(InvalidSubregionNameError) as e: 95 | _Downloader.validate_subregion_name('abc', ['ab']) 96 | assert ' -> ' in e 97 | 98 | subrgn_name = 'usa' 99 | subrgn_name_ = _Downloader.validate_subregion_name(subrgn_name) 100 | assert subrgn_name_ == 'United States of America' 101 | 102 | avail_subrgn_names = ['Greater London', 'Great Britain', 'Birmingham', 'Leeds'] 103 | 104 | subrgn_name = 'Britain' 105 | subrgn_name_ = _Downloader.validate_subregion_name(subrgn_name, avail_subrgn_names) 106 | assert subrgn_name_ == 'Great Britain' 107 | 108 | subrgn_name = 'london' 109 | subrgn_name_ = _Downloader.validate_subregion_name(subrgn_name, avail_subrgn_names) 110 | assert subrgn_name_ == 'Greater London' 111 | 112 | @staticmethod 113 | def test_validate_file_format(): 114 | with pytest.raises(InvalidFileFormatError) as e: 115 | file_fmt = 'abc' 116 | _Downloader.validate_file_format(file_fmt) # Raise an error 117 | assert "`osm_file_format='abc'` -> The input `osm_file_format` is unidentifiable." in e 118 | 119 | avail_file_fmts = ['.osm.pbf', '.shp.zip', '.osm.bz2'] 120 | 121 | file_fmt = 'pbf' 122 | assert _Downloader.validate_file_format(file_fmt, avail_file_fmts) == '.osm.pbf' 123 | 124 | file_fmt = 'shp' 125 | assert _Downloader.validate_file_format(file_fmt, avail_file_fmts) == '.shp.zip' 126 | 127 | @staticmethod 128 | def test_get_default_sub_path(): 129 | subrgn_name_ = 'London' 130 | dwnld_url = 'https://download.bbbike.org/osm/bbbike/London/London.osm.pbf' 131 | 132 | assert _Downloader.get_default_sub_path(subrgn_name_, dwnld_url) == '\\london' 133 | 134 | @staticmethod 135 | def test_make_subregion_dirname(): 136 | subrgn_name_ = 'England' 137 | assert _Downloader.make_subregion_dirname(subrgn_name_) == 'england' 138 | 139 | subrgn_name_ = 'Greater London' 140 | assert _Downloader.make_subregion_dirname(subrgn_name_) == 'greater-london' 141 | 142 | @staticmethod 143 | def test_get_subregion_download_url(): 144 | rslt = _Downloader.get_subregion_download_url('', '') 145 | assert rslt == ('', '') 146 | 147 | @staticmethod 148 | def test_get_valid_download_info(): 149 | d = _Downloader() 150 | 151 | subregion_name, osm_file_format = 'subregion_name', 'osm_file_format' 152 | 153 | valid_dwnld_info = d.get_valid_download_info(subregion_name, osm_file_format) 154 | subregion_name_, osm_filename, download_url, file_pathname = valid_dwnld_info 155 | assert subregion_name_ == '' 156 | assert osm_filename == '' 157 | assert download_url == '' 158 | assert os.path.relpath(file_pathname) == 'osm_data\\\\' 159 | 160 | d.download_dir = os.path.join(d.download_dir, '') 161 | _, _, _, file_pathname = d.get_valid_download_info(subregion_name, osm_file_format) 162 | assert os.path.relpath(file_pathname) == 'osm_data\\\\' 163 | 164 | download_dir = 'x-osm-pbf' 165 | _, _, _, file_pathname = d.get_valid_download_info( 166 | subregion_name, osm_file_format, download_dir) 167 | assert os.path.relpath(file_pathname) == 'x-osm-pbf\\' 168 | 169 | subregion_name, osm_file_format = '', '' 170 | _, osm_filename, _, file_pathname = d.get_valid_download_info( 171 | subregion_name=subregion_name, osm_file_format=osm_file_format) 172 | assert osm_filename is None and file_pathname is None 173 | 174 | @staticmethod 175 | def test_file_exists(capfd): 176 | d = _Downloader() 177 | 178 | subregion_name = '' 179 | osm_file_format = 'shp' 180 | rslt = d.file_exists(subregion_name=subregion_name, osm_file_format=osm_file_format) 181 | assert not rslt 182 | 183 | subregion_name, osm_file_format = '', '' 184 | rslt = d.file_exists( 185 | subregion_name=subregion_name, osm_file_format=osm_file_format, verbose=2) 186 | out, _ = capfd.readouterr() 187 | assert 'None data for "None" is not available' in out 188 | assert not rslt 189 | 190 | @staticmethod 191 | def test_file_exists_and_more(): 192 | subrgn_names, file_format = 'London', ".pbf" 193 | 194 | rslt = gfd.file_exists_and_more(subregion_names=subrgn_names, osm_file_format=file_format) 195 | assert rslt == (['Greater London'], '.osm.pbf', True, 'download', ['Greater London'], []) 196 | 197 | rslt = bbd.file_exists_and_more(subregion_names=subrgn_names, osm_file_format=file_format) 198 | assert rslt == (['London'], '.pbf', True, 'download', ['London'], []) 199 | 200 | subrgn_names = ['london', 'rutland'] 201 | rslt = gfd.file_exists_and_more(subregion_names=subrgn_names, osm_file_format=file_format) 202 | assert rslt == ( 203 | ['Greater London', 'Rutland'], '.osm.pbf', True, 'download', 204 | ['Greater London', 'Rutland'], 205 | []) 206 | 207 | subrgn_names = ['birmingham', 'leeds'] 208 | rslt = bbd.file_exists_and_more(subregion_names=subrgn_names, osm_file_format=file_format) 209 | assert rslt == ( 210 | ['Birmingham', 'Leeds'], '.pbf', True, 'download', ['Birmingham', 'Leeds'], []) 211 | 212 | @staticmethod 213 | def test_verify_download_dir(): 214 | d = _Downloader() 215 | assert os.path.relpath(d.download_dir) == 'osm_data' 216 | 217 | d.verify_download_dir(download_dir='tests\\osm_data', verify_download_dir=True) 218 | assert os.path.relpath(d.download_dir) == 'tests\\osm_data' 219 | 220 | 221 | class TestGeofabrikDownloader: 222 | 223 | @staticmethod 224 | def test_init(): 225 | assert gfd.NAME == 'Geofabrik' 226 | assert gfd.URL == 'https://download.geofabrik.de/' 227 | assert gfd.DOWNLOAD_INDEX_URL == 'https://download.geofabrik.de/index-v1.json' 228 | assert os.path.relpath(gfd.download_dir) == 'osm_data\\geofabrik' 229 | 230 | gfd_ = GeofabrikDownloader(download_dir="tests\\osm_data") 231 | assert os.path.relpath(gfd_.download_dir) == 'tests\\osm_data' 232 | 233 | assert isinstance(gfd.valid_subregion_names, set) 234 | assert isinstance(gfd.download_index, pd.DataFrame) 235 | assert isinstance(gfd.continent_tables, dict) 236 | assert isinstance(gfd.region_subregion_tier, dict) 237 | assert isinstance(gfd.having_no_subregions, list) 238 | assert isinstance(gfd.catalogue, pd.DataFrame) 239 | 240 | @staticmethod 241 | def test_get_raw_directory_index(capfd): 242 | raw_index = gfd.get_raw_directory_index(gfd.URL, verbose=True) 243 | out, _ = capfd.readouterr() 244 | assert out == \ 245 | "Collecting the raw directory index on 'https://download.geofabrik.de/' ... " \ 246 | "Failed.\n" \ 247 | "No raw directory index is available on the web page.\n" 248 | assert raw_index is None 249 | 250 | great_britain_url = 'https://download.geofabrik.de/europe/great-britain.html' 251 | raw_index = gfd.get_raw_directory_index(great_britain_url) 252 | assert isinstance(raw_index, pd.DataFrame) 253 | assert raw_index.columns.to_list() == ['file', 'date', 'size', 'metric_file_size', 'url'] 254 | 255 | @staticmethod 256 | def test_get_download_index(): 257 | geofabrik_dwnld_idx = gfd.get_download_index() 258 | 259 | assert isinstance(geofabrik_dwnld_idx, pd.DataFrame) 260 | assert geofabrik_dwnld_idx.columns.to_list() == [ 261 | 'id', 262 | 'parent', 263 | 'iso3166-1:alpha2', 264 | 'name', 265 | 'iso3166-2', 266 | 'geometry', 267 | '.osm.pbf', 268 | '.osm.bz2', 269 | '.shp.zip', 270 | 'pbf-internal', 271 | 'history', 272 | 'taginfo', 273 | 'updates'] 274 | 275 | @staticmethod 276 | def test_get_subregion_table(capfd): 277 | homepage = gfd.get_subregion_table(url=gfd.URL) 278 | 279 | assert homepage.columns.to_list() == [ 280 | 'subregion', 281 | 'subregion-url', 282 | '.osm.pbf', 283 | '.osm.pbf-size', 284 | '.shp.zip', 285 | '.osm.bz2'] 286 | 287 | great_britain_url = 'https://download.geofabrik.de/europe/great-britain.html' 288 | great_britain = gfd.get_subregion_table(great_britain_url) 289 | assert isinstance(great_britain, pd.DataFrame) 290 | 291 | antarctica_url = 'https://download.geofabrik.de/antarctica.html' 292 | antarctica = gfd.get_subregion_table(antarctica_url, verbose=True) 293 | out, _ = capfd.readouterr() 294 | assert out == 'Compiling information about subregions of "Antarctica" ... Failed.\n' 295 | assert antarctica is None 296 | 297 | antarctica2 = gfd.get_subregion_table(antarctica_url, verbose=2) 298 | out, _ = capfd.readouterr() 299 | assert out == \ 300 | "Compiling information about subregions of \"Antarctica\" ... Failed.\n" \ 301 | "No subregion data is available for \"Antarctica\" on " \ 302 | "Geofabrik's free download server.\n" 303 | assert antarctica2 is None 304 | 305 | @staticmethod 306 | def test_get_continent_tables(): 307 | continent_tables = gfd.get_continent_tables() 308 | assert isinstance(continent_tables, dict) 309 | 310 | asia_table = continent_tables['Asia'] 311 | assert asia_table.columns.to_list() == [ 312 | 'subregion', 313 | 'subregion-url', 314 | '.osm.pbf', 315 | '.osm.pbf-size', 316 | '.shp.zip', 317 | '.osm.bz2'] 318 | 319 | @staticmethod 320 | def test_get_region_subregion_tier(): 321 | rgn_subrgn_tier, no_subrgn_list = gfd.get_region_subregion_tier() 322 | assert isinstance(rgn_subrgn_tier, dict) 323 | assert isinstance(no_subrgn_list, list) 324 | 325 | @staticmethod 326 | def test_get_catalogue(): 327 | dwnld_catalog = gfd.get_catalogue() 328 | assert isinstance(dwnld_catalog, pd.DataFrame) 329 | assert len(dwnld_catalog) >= 474 330 | assert dwnld_catalog.columns.to_list() == [ 331 | 'subregion', 332 | 'subregion-url', 333 | '.osm.pbf', 334 | '.osm.pbf-size', 335 | '.shp.zip', 336 | '.osm.bz2'] 337 | 338 | @staticmethod 339 | def test_get_valid_subregion_names(): 340 | valid_subrgn_names = gfd.get_valid_subregion_names() 341 | assert isinstance(valid_subrgn_names, set) 342 | 343 | @staticmethod 344 | def test_validate_subregion_name(): 345 | input_subrgn_name = 'london' 346 | valid_subrgn_name = gfd.validate_subregion_name(subregion_name=input_subrgn_name) 347 | assert valid_subrgn_name == 'Greater London' 348 | 349 | input_subrgn_name = 'https://download.geofabrik.de/europe/great-britain.html' 350 | valid_subrgn_name = gfd.validate_subregion_name(subregion_name=input_subrgn_name) 351 | assert valid_subrgn_name == 'Great Britain' 352 | 353 | @staticmethod 354 | def test_validate_file_format(): 355 | input_file_format = ".pbf" 356 | valid_file_format = gfd.validate_file_format(osm_file_format=input_file_format) 357 | assert valid_file_format == '.osm.pbf' 358 | 359 | input_file_format = "shp" 360 | valid_file_format = gfd.validate_file_format(osm_file_format=input_file_format) 361 | assert valid_file_format == '.shp.zip' 362 | 363 | @staticmethod 364 | def test_get_subregion_download_url(): 365 | subrgn_name = 'England' 366 | file_format = ".pbf" 367 | valid_name, dwnld_link = gfd.get_subregion_download_url(subrgn_name, file_format) 368 | assert valid_name == 'England' 369 | assert dwnld_link == \ 370 | 'https://download.geofabrik.de/europe/great-britain/england-latest.osm.pbf' 371 | 372 | subrgn_name = 'britain' 373 | file_format = ".shp" 374 | valid_name, dwnld_link = gfd.get_subregion_download_url(subrgn_name, file_format) 375 | assert valid_name == 'Great Britain' 376 | assert dwnld_link is None 377 | 378 | @staticmethod 379 | def test_get_default_filename(capfd): 380 | subrgn_name, file_format = 'london', ".pbf" 381 | default_fn = gfd.get_default_filename(subrgn_name, file_format) 382 | assert default_fn == 'greater-london-latest.osm.pbf' 383 | 384 | subrgn_name, file_format = 'britain', ".shp" 385 | default_fn = gfd.get_default_filename(subrgn_name, file_format) 386 | out, _ = capfd.readouterr() 387 | assert out == 'No .shp.zip data is available to download for Great Britain.\n' 388 | assert default_fn is None 389 | 390 | @staticmethod 391 | def test_get_default_pathname(): 392 | subrgn_name, file_format = 'london', ".pbf" 393 | 394 | pathname, filename = gfd.get_default_pathname(subrgn_name, file_format) 395 | assert os.path.relpath(os.path.dirname(pathname)) == \ 396 | 'osm_data\\geofabrik\\europe\\great-britain\\england\\greater-london' 397 | assert filename == 'greater-london-latest.osm.pbf' 398 | 399 | @staticmethod 400 | def test_get_subregions(): 401 | all_subrgn_names = gfd.get_subregions() 402 | assert isinstance(all_subrgn_names, list) 403 | 404 | e_na_subrgn_names = gfd.get_subregions('england', 'n america') 405 | assert isinstance(e_na_subrgn_names, list) 406 | 407 | na_subrgn_names = gfd.get_subregions('n america', deep=True) 408 | assert isinstance(na_subrgn_names, list) 409 | 410 | gb_subrgn_names = gfd.get_subregions('britain') 411 | gb_subrgn_names_ = gfd.get_subregions('britain', deep=True) 412 | assert len(gb_subrgn_names_) >= len(gb_subrgn_names) 413 | 414 | @staticmethod 415 | def test_specify_sub_download_dir(): 416 | subrgn_name = 'london' 417 | file_format = ".pbf" 418 | 419 | dwnld_dir = gfd.specify_sub_download_dir(subrgn_name, file_format) 420 | assert os.path.dirname(os.path.relpath(dwnld_dir)) == \ 421 | 'osm_data\\geofabrik\\europe\\great-britain\\england\\greater-london' 422 | 423 | dwnld_dir = "tests\\osm_data" 424 | 425 | subrgn_name = 'britain' 426 | file_format = ".shp" 427 | 428 | dwnld_pathname = gfd.specify_sub_download_dir(subrgn_name, file_format, dwnld_dir) 429 | assert os.path.relpath(dwnld_pathname) == 'tests\\osm_data\\great-britain-shp-zip' 430 | 431 | gfd_ = GeofabrikDownloader(download_dir=dwnld_dir) 432 | dwnld_pathname_ = gfd_.specify_sub_download_dir(subrgn_name, file_format) 433 | assert os.path.relpath(dwnld_pathname_) == \ 434 | 'tests\\osm_data\\europe\\great-britain\\great-britain-shp-zip' 435 | 436 | @staticmethod 437 | def test_get_valid_download_info(): 438 | subrgn_name = 'london' 439 | file_format = "pbf" 440 | 441 | info1 = gfd.get_valid_download_info(subrgn_name, file_format) 442 | valid_subrgn_name, pbf_filename, dwnld_url, path_to_pbf = info1 443 | 444 | assert valid_subrgn_name == 'Greater London' 445 | assert pbf_filename == 'greater-london-latest.osm.pbf' 446 | assert dwnld_url == \ 447 | 'https://download.geofabrik.de/europe/great-britain/england/' \ 448 | 'greater-london-latest.osm.pbf' 449 | assert os.path.relpath(path_to_pbf) == \ 450 | 'osm_data\\geofabrik\\europe\\great-britain\\england\\greater-london\\' \ 451 | 'greater-london-latest.osm.pbf' 452 | 453 | dwnld_dir = "tests\\osm_data" 454 | 455 | info2 = gfd.get_valid_download_info(subrgn_name, file_format, dwnld_dir) 456 | _, _, _, path_to_pbf2 = info2 457 | 458 | assert os.path.relpath(path_to_pbf2) == \ 459 | 'tests\\osm_data\\greater-london\\greater-london-latest.osm.pbf' 460 | 461 | gfd_ = GeofabrikDownloader(download_dir=dwnld_dir) 462 | 463 | info3 = gfd_.get_valid_download_info(subrgn_name, file_format) 464 | _, _, _, path_to_pbf3 = info3 465 | 466 | assert os.path.relpath(path_to_pbf3) == \ 467 | 'tests\\osm_data\\europe\\great-britain\\england\\greater-london\\' \ 468 | 'greater-london-latest.osm.pbf' 469 | 470 | @staticmethod 471 | def test_download_osm_data(capfd): 472 | gfd_ = GeofabrikDownloader() 473 | 474 | subrgn_names = ['london', 'rutland'] 475 | file_format = ".pbf" 476 | gfd_.download_osm_data( 477 | subregion_names=subrgn_names, osm_file_format=file_format, verbose=True, 478 | confirmation_required=False) 479 | out, _ = capfd.readouterr() 480 | assert "Downloading " in out and "Done." in out 481 | assert len(gfd_.data_paths) == 2 482 | assert os.path.relpath(gfd_.download_dir) == 'osm_data\\geofabrik' 483 | 484 | dwnld_dir = os.path.dirname(gfd_.download_dir) 485 | 486 | region_name = 'west midlands' 487 | file_format = ".shp" 488 | temp_dwnld_dir = tempfile.TemporaryDirectory().name 489 | 490 | gfd_.download_osm_data( 491 | subregion_names=region_name, osm_file_format=file_format, download_dir=temp_dwnld_dir, 492 | confirmation_required=False) 493 | assert len(gfd_.data_paths) == 3 494 | assert gfd_.data_paths[-1] == \ 495 | f'{temp_dwnld_dir}\\west-midlands\\west-midlands-latest-free.shp.zip' 496 | assert gfd_.download_dir == temp_dwnld_dir 497 | assert os.path.relpath(gfd_.cdd()) == 'osm_data\\geofabrik' 498 | 499 | delete_dir([dwnld_dir, temp_dwnld_dir], confirmation_required=False) 500 | 501 | # @staticmethod 502 | # def test_download_subregion_data(): 503 | # gfd_ = GeofabrikDownloader() 504 | # 505 | # subrgn_name = 'England' 506 | # file_format = ".pbf" 507 | # dwnld_dir = "tests\\osm_data" 508 | # 509 | # dwnld_file_pathnames = gfd_.download_subregion_data( 510 | # subrgn_name, file_format, download_dir=dwnld_dir, update=True, verbose=True, 511 | # ret_download_path=True, confirmation_required=False) 512 | # 513 | # assert len(dwnld_file_pathnames) >= 47 514 | # assert os.path.commonpath(dwnld_file_pathnames) == gfd_.download_dir 515 | # 516 | # delete_dir(gfd_.download_dir, confirmation_required=False) 517 | 518 | 519 | class TestBBBikeDownloader: 520 | 521 | @staticmethod 522 | def test_init(): 523 | assert bbd.NAME == 'BBBike' 524 | assert bbd.LONG_NAME == 'BBBike exports of OpenStreetMap data' 525 | assert bbd.URL == 'https://download.bbbike.org/osm/bbbike/' 526 | assert os.path.relpath(bbd.download_dir) == 'osm_data\\bbbike' 527 | 528 | bbd_ = BBBikeDownloader(download_dir="tests\\osm_data") 529 | assert os.path.relpath(bbd_.download_dir) == 'tests\\osm_data' 530 | 531 | assert isinstance(bbd.valid_subregion_names, list) 532 | assert isinstance(bbd.subregion_coordinates, pd.DataFrame) 533 | assert isinstance(bbd.subregion_index, pd.DataFrame) 534 | assert isinstance(bbd.catalogue, dict) 535 | 536 | @staticmethod 537 | def test_get_names_of_cities(): 538 | bbbike_cities = bbd.get_names_of_cities() 539 | assert isinstance(bbbike_cities, list) 540 | 541 | @staticmethod 542 | def test_get_coordinates_of_cities(): 543 | coords_of_cities = bbd.get_coordinates_of_cities() 544 | 545 | assert isinstance(coords_of_cities, pd.DataFrame) 546 | assert coords_of_cities.columns.to_list() == [ 547 | 'city', 548 | 'real_name', 549 | 'pref._language', 550 | 'local_language', 551 | 'country', 552 | 'area_or_continent', 553 | 'population', 554 | 'step', 555 | 'other_cities', 556 | 'll_longitude', 557 | 'll_latitude', 558 | 'ur_longitude', 559 | 'ur_latitude'] 560 | 561 | @staticmethod 562 | def test_get_subregion_index(): 563 | subrgn_idx = bbd.get_subregion_index() 564 | 565 | assert isinstance(subrgn_idx, pd.DataFrame) 566 | assert subrgn_idx.columns.to_list() == ['name', 'last_modified', 'url'] 567 | 568 | @staticmethod 569 | def test_get_valid_subregion_names(): 570 | subrgn_names = bbd.get_valid_subregion_names() 571 | 572 | assert isinstance(subrgn_names, list) 573 | 574 | @staticmethod 575 | def test_validate_subregion_name(): 576 | subrgn_name = 'birmingham' 577 | 578 | valid_name = bbd.validate_subregion_name(subregion_name=subrgn_name) 579 | assert valid_name == 'Birmingham' 580 | 581 | @staticmethod 582 | def test_get_subregion_catalogue(capfd): 583 | subrgn_name = 'birmingham' 584 | 585 | bham_dwnld_cat = bbd.get_subregion_catalogue( 586 | subregion_name=subrgn_name, confirmation_required=False, verbose=True) 587 | out, _ = capfd.readouterr() 588 | assert 'Compiling the data of a download catalogue for "Birmingham" ... Done.\n' == out 589 | assert isinstance(bham_dwnld_cat, pd.DataFrame) 590 | assert bham_dwnld_cat.columns.to_list() == [ 591 | 'filename', 'url', 'data_type', 'size', 'last_update'] 592 | 593 | @staticmethod 594 | def test_get_catalogue(): 595 | bbbike_catalogue = bbd.get_catalogue() 596 | assert list(bbbike_catalogue.keys()) == ['FileFormat', 'DataType', 'Catalogue'] 597 | 598 | catalogue = bbbike_catalogue['Catalogue'] 599 | assert isinstance(catalogue, dict) 600 | 601 | bham_catalogue = catalogue['Birmingham'] 602 | assert isinstance(bham_catalogue, pd.DataFrame) 603 | 604 | @staticmethod 605 | def test_validate_file_format(): 606 | valid_file_format = bbd.validate_file_format(osm_file_format='PBF') 607 | assert valid_file_format == '.pbf' 608 | 609 | valid_file_format = bbd.validate_file_format(osm_file_format='.osm.pbf') 610 | assert valid_file_format == '.pbf' 611 | 612 | @staticmethod 613 | def test_get_subregion_download_url(): 614 | subrgn_name = 'birmingham' 615 | file_format = "pbf" 616 | 617 | subrgn_name_, dwnld_url = bbd.get_subregion_download_url(subrgn_name, file_format) 618 | assert subrgn_name_ == 'Birmingham' 619 | assert dwnld_url == 'https://download.bbbike.org/osm/bbbike/Birmingham/Birmingham.osm.pbf' 620 | 621 | file_format = "csv.xz" 622 | subrgn_name_, dwnld_url = bbd.get_subregion_download_url(subrgn_name, file_format) 623 | 624 | assert subrgn_name_ == 'Birmingham' 625 | assert dwnld_url == \ 626 | 'https://download.bbbike.org/osm/bbbike/Birmingham/Birmingham.osm.csv.xz' 627 | 628 | @staticmethod 629 | def test_get_valid_download_info(): 630 | subrgn_name = 'birmingham' 631 | file_format = "pbf" 632 | 633 | info = bbd.get_valid_download_info(subrgn_name, file_format) 634 | valid_subrgn_name, pbf_filename, dwnld_url, pbf_pathname = info 635 | 636 | assert valid_subrgn_name == 'Birmingham' 637 | assert pbf_filename == 'Birmingham.osm.pbf' 638 | assert dwnld_url == 'https://download.bbbike.org/osm/bbbike/Birmingham/Birmingham.osm.pbf' 639 | assert os.path.relpath(pbf_pathname) == 'osm_data\\bbbike\\birmingham\\Birmingham.osm.pbf' 640 | 641 | bbd_ = BBBikeDownloader(download_dir="tests\\osm_data") 642 | _, _, _, pbf_pathname = bbd_.get_valid_download_info(subrgn_name, file_format) 643 | assert os.path.relpath(pbf_pathname) == 'tests\\osm_data\\birmingham\\Birmingham.osm.pbf' 644 | 645 | @staticmethod 646 | def test_file_exists(): 647 | subrgn_name = 'birmingham' 648 | file_format = ".pbf" 649 | dwnld_dir = "tests\\osm_data" 650 | 651 | pbf_exists = bbd.file_exists(subrgn_name, file_format, dwnld_dir) 652 | assert not pbf_exists 653 | 654 | # @staticmethod 655 | # def test_download_subregion_data(capfd): 656 | # subrgn_name = 'leeds' 657 | # dwnld_dir = "tests\\osm_data" 658 | # 659 | # bbd.download_subregion_data( 660 | # subregion_name=subrgn_name, download_dir=dwnld_dir, ret_download_path=True, 661 | # verbose=True, confirmation_required=False) 662 | # out, _ = capfd.readouterr() 663 | # assert 'Check out the downloaded OSM data at "tests\\osm_data\\leeds\\".' in out 664 | 665 | @staticmethod 666 | def test_download_osm_data(): 667 | subrgn_name = 'London' 668 | file_format = "pbf" 669 | 670 | bbd.download_osm_data(subrgn_name, file_format, confirmation_required=False) 671 | assert len(bbd.data_paths) == 1 672 | assert os.path.relpath(bbd.data_paths[0]) == 'osm_data\\bbbike\\london\\London.osm.pbf' 673 | 674 | london_dwnld_dir = os.path.relpath(bbd.download_dir) 675 | assert london_dwnld_dir == 'osm_data\\bbbike' 676 | 677 | subrgn_names = ['leeds', 'birmingham'] 678 | dwnld_dir = "tests\\osm_data" 679 | 680 | dwnld_paths = bbd.download_osm_data( 681 | subrgn_names, file_format, dwnld_dir, confirmation_required=False, 682 | ret_download_path=True) 683 | assert len(dwnld_paths) == 2 684 | assert len(bbd.data_paths) == 3 685 | assert os.path.relpath(bbd.download_dir) == os.path.relpath(dwnld_dir) 686 | assert os.path.relpath(os.path.commonpath(dwnld_paths)) == 'tests\\osm_data' 687 | 688 | delete_dir([os.path.dirname(london_dwnld_dir), dwnld_dir], confirmation_required=False) 689 | 690 | 691 | if __name__ == '__main__': 692 | pytest.main() 693 | -------------------------------------------------------------------------------- /tests/test_errors.py: -------------------------------------------------------------------------------- 1 | """Test the module :py:mod:`pydriosm.errors`.""" 2 | 3 | import pytest 4 | 5 | 6 | class TestInvalidSubregionNameError: 7 | 8 | @staticmethod 9 | def test_error(): 10 | from pydriosm.errors import InvalidSubregionNameError 11 | 12 | msg = InvalidSubregionNameError('abc').message 13 | assert 'The input of `subregion_name` is not recognizable.' in msg 14 | 15 | msg = InvalidSubregionNameError('abc').__str__() 16 | assert ' -> ' in msg 17 | 18 | msg = InvalidSubregionNameError('abc', msg=1).message 19 | assert '1)' in msg and '2)' in msg 20 | 21 | 22 | class TestInvalidFileFormatError: 23 | 24 | @staticmethod 25 | def test_error(): 26 | from pydriosm.errors import InvalidFileFormatError 27 | 28 | msg = InvalidFileFormatError('abc').message 29 | assert 'The input `osm_file_format` is unidentifiable.' in msg 30 | 31 | msg = InvalidFileFormatError('abc').__str__() 32 | assert ' -> ' in msg 33 | 34 | msg = InvalidFileFormatError('abc', valid_file_formats={'valid_file_formats'}).message 35 | assert 'Valid options include:' in msg 36 | 37 | 38 | class TestOtherTagsReformatError: 39 | 40 | @staticmethod 41 | def test_error(): 42 | from pydriosm.errors import OtherTagsReformatError 43 | 44 | msg = OtherTagsReformatError('abc').message 45 | assert 'Failed to reformat the `other_tags`.' in msg 46 | 47 | msg = OtherTagsReformatError('abc').__str__() 48 | assert ' -> ' in msg 49 | 50 | 51 | if __name__ == '__main__': 52 | pytest.main() 53 | -------------------------------------------------------------------------------- /tests/test_ios.py: -------------------------------------------------------------------------------- 1 | """Test the module :py:mod:`pydriosm.ios`.""" 2 | 3 | import pytest 4 | 5 | 6 | def test_get_default_layer_name(): 7 | from pydriosm.ios.utils import get_default_layer_name 8 | 9 | lyr_name = get_default_layer_name(schema_name='point') 10 | assert lyr_name == 'points' 11 | 12 | lyr_name = get_default_layer_name(schema_name='land') 13 | assert lyr_name == 'landuse' 14 | 15 | 16 | def test_validate_schema_names(): 17 | from pydriosm.ios.utils import validate_schema_names 18 | 19 | valid_names = validate_schema_names() 20 | assert valid_names == [] 21 | 22 | input_schema_names = ['point', 'polygon'] 23 | valid_names = validate_schema_names(input_schema_names) 24 | assert valid_names == ['point', 'polygon'] 25 | 26 | valid_names = validate_schema_names(input_schema_names, schema_named_as_layer=True) 27 | assert valid_names == ['points', 'multipolygons'] 28 | 29 | 30 | def test_validate_table_name(): 31 | from pydriosm.ios.utils import validate_table_name 32 | 33 | subrgn_name = 'greater london' 34 | valid_table_name = validate_table_name(subrgn_name) 35 | assert valid_table_name == 'greater london' 36 | 37 | subrgn_name = 'Llanfairpwllgwyngyllgogerychwyrndrobwllllantysiliogogogoch, Wales' 38 | valid_table_name = validate_table_name(subrgn_name, sub_space='_') 39 | assert valid_table_name == 'Llanfairpwllgwyngyllgogerychwyrndrobwllllantysiliogogogoch_W..' 40 | 41 | 42 | # from pydriosm.ios import PostgresOSM 43 | # from pydriosm.downloader import GeofabrikDownloader, BBBikeDownloader 44 | # from pydriosm.reader import GeofabrikReader, BBBikeReader 45 | # 46 | # 47 | # osmdb = PostgresOSM(database_name='osmdb_test') 48 | # 49 | # 50 | # class TestPostgresOSM: 51 | # 52 | # @staticmethod 53 | # def test_init(): 54 | # assert osmdb.data_source == 'Geofabrik' 55 | # assert osmdb.name == 'Geofabrik OpenStreetMap data extracts' 56 | # assert osmdb.url == 'https://download.geofabrik.de/' 57 | # assert isinstance(osmdb.downloader, GeofabrikDownloader) 58 | # assert isinstance(osmdb.reader, GeofabrikReader) 59 | # 60 | # # Change the data source 61 | # assert osmdb.data_source == 'BBBike' 62 | # assert osmdb.name == 'BBBike exports of OpenStreetMap data' 63 | # assert osmdb.url == 'https://download.bbbike.org/osm/bbbike/' 64 | # assert isinstance(osmdb.downloader, BBBikeDownloader) 65 | # assert isinstance(osmdb.reader, BBBikeReader) 66 | 67 | 68 | if __name__ == '__main__': 69 | pytest.main() 70 | -------------------------------------------------------------------------------- /tests/test_reader.py: -------------------------------------------------------------------------------- 1 | """Test the module :py:mod:`pydriosm.reader`.""" 2 | 3 | import glob 4 | import os 5 | import shutil 6 | 7 | import pandas as pd 8 | import pytest 9 | import shapely.geometry 10 | from pyhelpers.store import load_pickle 11 | 12 | from pydriosm.reader import PBFReadParse, SHPReadParse, Transformer 13 | from pydriosm.reader._reader import _Reader 14 | 15 | 16 | class TestTransformer: 17 | TEST_POINT_1 = { 18 | 'type': 'Point', 19 | 'coordinates': [-0.5134241, 52.6555853] 20 | } 21 | 22 | TEST_POINT_2 = { 23 | 'type': 'Feature', 24 | 'geometry': { 25 | 'type': 'Point', 26 | 'coordinates': [-0.5134241, 52.6555853] 27 | }, 28 | 'properties': { 29 | 'osm_id': '488432', 30 | 'name': None, 31 | 'barrier': None, 32 | 'highway': None, 33 | 'ref': None, 34 | 'address': None, 35 | 'is_in': None, 36 | 'place': None, 37 | 'man_made': None, 38 | 'other_tags': '"odbl"=>"clean"' 39 | }, 40 | 'id': 488432 41 | } 42 | 43 | TEST_COLLECTION_1 = { 44 | 'type': 'GeometryCollection', 45 | 'geometries': [ 46 | {'type': 'Point', 'coordinates': [-0.5096176, 52.6605168]}, 47 | {'type': 'Point', 'coordinates': [-0.5097337, 52.6605812]} 48 | ] 49 | } 50 | 51 | TEST_COLLECTION_2 = { 52 | 'type': 'Feature', 53 | 'geometry': { 54 | 'type': 'GeometryCollection', 55 | 'geometries': [ 56 | {'type': 'Point', 'coordinates': [-0.5096176, 52.6605168]}, 57 | {'type': 'Point', 'coordinates': [-0.5097337, 52.6605812]}] 58 | }, 59 | 'properties': { 60 | 'osm_id': '256254', 61 | 'name': 'Fife Close', 62 | 'type': 'site', 63 | 'other_tags': '"naptan:StopAreaCode"=>"270G02701525"' 64 | }, 65 | 'id': 256254 66 | } 67 | 68 | @staticmethod 69 | def test_point_as_polygon(): 70 | geometry = { 71 | 'type': 'MultiPolygon', 72 | 'coordinates': [[[[-0.6920145, 52.6753268], [-0.6920145, 52.6753268]]]] 73 | } 74 | mp_coords = geometry['coordinates'] 75 | 76 | mp_coords_ = Transformer.point_as_polygon(mp_coords) 77 | assert mp_coords_ == [ 78 | [[[-0.6920145, 52.6753268], 79 | [-0.6920145, 52.6753268], 80 | [-0.6920145, 52.6753268]]]] 81 | 82 | @classmethod 83 | def test_transform_unitary_geometry(cls): 84 | g1_dat = cls.TEST_POINT_1.copy() 85 | g1_data = Transformer.transform_unitary_geometry(g1_dat) 86 | assert isinstance(g1_data, shapely.geometry.Point) 87 | assert g1_data.wkt == 'POINT (-0.5134241 52.6555853)' 88 | 89 | g2_dat = cls.TEST_POINT_2.copy() 90 | g2_data = Transformer.transform_unitary_geometry(g2_dat, mode=2) 91 | 92 | assert isinstance(g2_data, dict) 93 | assert list(g2_data.keys()) == ['type', 'geometry', 'properties', 'id'] 94 | assert g2_data['geometry'] == 'POINT (-0.5134241 52.6555853)' 95 | 96 | @classmethod 97 | def test_transform_geometry_collection(cls): 98 | g1_dat_ = cls.TEST_COLLECTION_1.copy() 99 | g1_dat = g1_dat_['geometries'] 100 | g1_data = Transformer.transform_geometry_collection(g1_dat) 101 | assert isinstance(g1_data, shapely.geometry.base.BaseGeometry) 102 | assert (g1_data.wkt == 103 | 'GEOMETRYCOLLECTION (POINT (-0.5096176 52.6605168), POINT (-0.5097337 52.6605812))') 104 | 105 | g2_dat = cls.TEST_COLLECTION_2.copy() 106 | g2_data = Transformer.transform_geometry_collection(g2_dat, mode=2) 107 | assert isinstance(g2_data, dict) 108 | assert list(g2_data.keys()) == ['type', 'geometry', 'properties', 'id'] 109 | assert g2_data['geometry'] == \ 110 | 'GEOMETRYCOLLECTION (POINT (-0.5096176 52.6605168), POINT (-0.5097337 52.6605812))' 111 | 112 | @classmethod 113 | def test_transform_geometry(cls): 114 | lyr_name = 'points' 115 | dat_ = cls.TEST_POINT_2.copy() 116 | 117 | lyr_data = pd.DataFrame.from_dict(dat_, orient='index').T 118 | 119 | geom_dat = Transformer.transform_geometry(layer_data=lyr_data, layer_name=lyr_name) 120 | assert isinstance(geom_dat, pd.Series) 121 | assert geom_dat.values[0].wkt == 'POINT (-0.5134241 52.6555853)' 122 | 123 | @staticmethod 124 | def test_transform_other_tags(): 125 | other_tags_dat = Transformer.transform_other_tags(other_tags='"odbl"=>"clean"') 126 | assert other_tags_dat == {'odbl': 'clean'} 127 | 128 | @staticmethod 129 | def test_update_other_tags(): 130 | prop_dat = { 131 | 'properties': { 132 | 'osm_id': '488432', 133 | 'name': None, 134 | 'barrier': None, 135 | 'highway': None, 136 | 'ref': None, 137 | 'address': None, 138 | 'is_in': None, 139 | 'place': None, 140 | 'man_made': None, 141 | 'other_tags': '"odbl"=>"clean"' 142 | }, 143 | } 144 | prop_dat_ = Transformer.update_other_tags(prop_dat['properties']) 145 | assert prop_dat_ == { 146 | 'osm_id': '488432', 147 | 'name': None, 148 | 'barrier': None, 149 | 'highway': None, 150 | 'ref': None, 151 | 'address': None, 152 | 'is_in': None, 153 | 'place': None, 154 | 'man_made': None, 155 | 'other_tags': {'odbl': 'clean'} 156 | } 157 | 158 | 159 | class TestPBFReadParse: 160 | path_to_osm_pbf = "tests\\data\\rutland\\rutland-latest.osm.pbf" 161 | 162 | @staticmethod 163 | def test_get_pbf_layer_geom_types(): 164 | pbf_layer_geom_dict = PBFReadParse.get_pbf_layer_geom_types(shape_name=True) 165 | assert pbf_layer_geom_dict == { 166 | 'points': 'Point', 167 | 'lines': 'LineString', 168 | 'multilinestrings': 'MultiLineString', 169 | 'multipolygons': 'MultiPolygon', 170 | 'other_relations': 'GeometryCollection'} 171 | 172 | @staticmethod 173 | @pytest.mark.parametrize('layer_name', ['points', 'other_relations']) 174 | @pytest.mark.parametrize('dat_id', [1, 2]) 175 | @pytest.mark.parametrize('parse_geometry', [True, False]) 176 | @pytest.mark.parametrize('parse_properties', [True, False]) 177 | @pytest.mark.parametrize('parse_other_tags', [True, False]) 178 | def test_transform_pbf_layer_field(layer_name, dat_id, parse_geometry, parse_properties, 179 | parse_other_tags): 180 | layer_data = load_pickle(f"tests\\data\\rutland\\{layer_name}_{dat_id}.pkl") 181 | lyr_dat = PBFReadParse.transform_pbf_layer_field(layer_data=layer_data, layer_name=layer_name) 182 | 183 | assert isinstance(lyr_dat, (pd.Series, pd.DataFrame)) 184 | 185 | @pytest.mark.parametrize('readable', [False, True]) 186 | @pytest.mark.parametrize('expand', [False, True]) 187 | @pytest.mark.parametrize('number_of_chunks', [None, 5]) 188 | def test_read_pbf(self, readable, expand, number_of_chunks): 189 | rutland_pbf = PBFReadParse.read_pbf( 190 | pbf_pathname=self.path_to_osm_pbf, 191 | readable=readable, 192 | expand=expand, 193 | number_of_chunks=number_of_chunks) 194 | 195 | assert isinstance(rutland_pbf, dict) 196 | assert list(rutland_pbf.keys()) == [ 197 | 'points', 'lines', 'multilinestrings', 'multipolygons', 'other_relations'] 198 | 199 | 200 | class TestSHPReadParse: 201 | path_to_shp_zip = "tests\\data\\rutland\\rutland-latest-free.shp.zip" 202 | extract_to_dir = "tests\\data\\rutland\\temp" 203 | 204 | @staticmethod 205 | def test_validate_shp_layer_names(): 206 | assert SHPReadParse.validate_shp_layer_names(None) == [] 207 | assert SHPReadParse.validate_shp_layer_names('point') == ['points'] 208 | assert SHPReadParse.validate_shp_layer_names(['point', 'land']) == ['points', 'landuse'] 209 | assert len(SHPReadParse.validate_shp_layer_names('all')) >= 13 210 | 211 | @staticmethod 212 | def test_find_shp_layer_name(): 213 | assert SHPReadParse.find_shp_layer_name("") is None 214 | assert SHPReadParse.find_shp_layer_name("gis_osm_railways_free_1.shp") == 'railways' 215 | assert SHPReadParse.find_shp_layer_name("gis_osm_transport_a_free_1.shp") == 'transport' 216 | 217 | def test_unzip_shp_zip(self): 218 | rutland_shp_dir = SHPReadParse.unzip_shp_zip( 219 | self.path_to_shp_zip, extract_to=self.extract_to_dir, layer_names='railways', 220 | verbose=True, ret_extract_dir=True) 221 | assert rutland_shp_dir == self.extract_to_dir 222 | 223 | lyr_names = ['railways', 'transport', 'traffic'] 224 | dirs_of_layers = SHPReadParse.unzip_shp_zip( 225 | self.path_to_shp_zip, extract_to=self.extract_to_dir, layer_names=lyr_names, 226 | separate=True, verbose=2, ret_extract_dir=True) 227 | assert os.path.relpath(os.path.commonpath(dirs_of_layers)) == self.extract_to_dir 228 | assert all(x in lyr_names for x in map(os.path.basename, dirs_of_layers)) 229 | 230 | rutland_shp_dir = SHPReadParse.unzip_shp_zip( 231 | self.path_to_shp_zip, extract_to=self.extract_to_dir, ret_extract_dir=True, verbose=True) 232 | layer_names = set( 233 | filter(None, map(SHPReadParse.find_shp_layer_name, os.listdir(rutland_shp_dir)))) 234 | assert all(x in SHPReadParse.LAYER_NAMES for x in layer_names) 235 | 236 | def test_read_shp(self): 237 | rutland_shp_dir = SHPReadParse.unzip_shp_zip( 238 | self.path_to_shp_zip, extract_to=self.extract_to_dir, ret_extract_dir=True) 239 | path_to_railways_shp = glob.glob(os.path.join(rutland_shp_dir, "*railways*.shp"))[0] 240 | 241 | rutland_railways = SHPReadParse.read_shp(path_to_railways_shp) 242 | assert isinstance(rutland_railways, pd.DataFrame) 243 | 244 | rutland_railways = SHPReadParse.read_shp(path_to_railways_shp, emulate_gpd=True) 245 | assert isinstance(rutland_railways, pd.DataFrame) 246 | 247 | rutland_railways_ = SHPReadParse.read_shp(path_to_railways_shp, engine='geopandas') 248 | 249 | railways_data = [rutland_railways, rutland_railways_] 250 | geom1, geom2 = map(lambda x: x['geometry'].map(lambda y: y.wkt), railways_data) 251 | assert geom1.equals(geom2) 252 | 253 | def test_read_layer_shps(self): 254 | rutland_shp_dir = SHPReadParse.unzip_shp_zip( 255 | self.path_to_shp_zip, extract_to=self.extract_to_dir, ret_extract_dir=True) 256 | rutland_railways_shp_path = os.path.join(rutland_shp_dir, "gis_osm_railways_free_1.shp") 257 | 258 | london_railways_shp = SHPReadParse.read_layer_shps(shp_pathnames=rutland_railways_shp_path) 259 | assert isinstance(london_railways_shp, pd.DataFrame) 260 | 261 | railways_rail_shp, railways_rail_shp_path = SHPReadParse.read_layer_shps( 262 | rutland_railways_shp_path, feature_names='rail', save_feat_shp=True, 263 | ret_feat_shp_path=True) 264 | assert isinstance(railways_rail_shp, pd.DataFrame) 265 | assert all(os.path.isfile(x) for x in railways_rail_shp_path) 266 | 267 | shutil.rmtree(self.extract_to_dir) 268 | 269 | 270 | class TestReader: 271 | 272 | @staticmethod 273 | def test_init(): 274 | r = _Reader() 275 | 276 | assert r.NAME == 'OSM Reader' 277 | assert isinstance(r.SHP, type) and r.SHP == SHPReadParse 278 | 279 | @staticmethod 280 | def test_cdd(): 281 | assert os.path.relpath(_Reader.cdd()) == 'osm_data' 282 | 283 | @staticmethod 284 | def test_data_dir(): 285 | from pydriosm.downloader import GeofabrikDownloader, BBBikeDownloader 286 | 287 | r = _Reader() 288 | assert os.path.relpath(r.data_dir) == 'osm_data' 289 | 290 | r = _Reader(downloader=GeofabrikDownloader) 291 | assert os.path.relpath(r.data_dir) == 'osm_data\\geofabrik' 292 | 293 | r = _Reader(downloader=BBBikeDownloader) 294 | assert os.path.relpath(r.data_dir) == 'osm_data\\bbbike' 295 | 296 | @staticmethod 297 | def test_data_paths(): 298 | r = _Reader() 299 | assert r.data_paths == [] 300 | 301 | 302 | if __name__ == '__main__': 303 | pytest.main() 304 | -------------------------------------------------------------------------------- /tests/test_updater.py: -------------------------------------------------------------------------------- 1 | """Test the module :py:mod:`pydriosm._updater`.""" 2 | 3 | import pytest 4 | 5 | 6 | def test__update_prepacked_data(monkeypatch, capfd): 7 | from pydriosm._updater import _update_prepacked_data 8 | 9 | monkeypatch.setattr('builtins.input', lambda _: "Yes") 10 | _update_prepacked_data(verbose=True) 11 | out, _ = capfd.readouterr() 12 | assert "Done." in out and "Update finished." in out and "Failed." not in out 13 | 14 | 15 | if __name__ == '__main__': 16 | pytest.main() 17 | -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- 1 | """Test the module :py:mod:`pydriosm.utils`.""" 2 | 3 | import os 4 | import shutil 5 | import types 6 | 7 | import pytest 8 | 9 | 10 | @pytest.mark.parametrize('mkdir', [False, True]) 11 | def test__cdd(mkdir): 12 | from pydriosm.utils import _cdd 13 | 14 | data_dir = "pytest_pydriosm_data" 15 | pathname = _cdd(data_dir=data_dir, mkdir=mkdir) 16 | assert os.path.basename(pathname) == data_dir 17 | if mkdir: 18 | os.rmdir(pathname) 19 | 20 | data_dir = "pytest_pydriosm_data\\tests.ext" 21 | pathname = _cdd(data_dir=data_dir, mkdir=mkdir) 22 | assert os.path.basename(pathname) == "tests.ext" 23 | if mkdir: 24 | shutil.rmtree(os.path.dirname(pathname)) 25 | 26 | 27 | def test_cdd_geofabrik(): 28 | from pydriosm.utils import cdd_geofabrik 29 | 30 | assert os.path.relpath(cdd_geofabrik()) == 'osm_geofabrik' 31 | 32 | 33 | def test_cdd_bbbike(): 34 | from pydriosm.utils import cdd_bbbike 35 | 36 | assert os.path.relpath(cdd_bbbike()) == 'osm_bbbike' 37 | 38 | 39 | def test_first_unique(): 40 | from pydriosm.utils import first_unique 41 | 42 | list_example1 = [1, 2, 2, 3, 4, 5, 6, 6, 2, 3, 1, 6] 43 | assert list(first_unique(list_example1)) == [1, 2, 3, 4, 5, 6] 44 | 45 | list_example2 = [6, 1, 2, 2, 3, 4, 5, 6, 6, 2, 3, 1] 46 | assert list(first_unique(list_example2)) == [6, 1, 2, 3, 4, 5] 47 | 48 | 49 | @pytest.mark.parametrize('engine', [None, 'ujson', 'orjson', 'rapidjson', 'json']) 50 | def test_check_json_engine(engine): 51 | from pydriosm.utils import check_json_engine 52 | 53 | result = check_json_engine(engine) 54 | 55 | assert isinstance(result, types.ModuleType) 56 | 57 | 58 | def test_remove_osm_file(capfd): 59 | from pydriosm.utils import remove_osm_file 60 | 61 | path_to_pseudo_pbf_file = os.path.join("tests\\data\\pseudo\\pseudo.osm.pbf") 62 | 63 | remove_osm_file(path_to_pseudo_pbf_file, verbose=True) 64 | out, _ = capfd.readouterr() 65 | assert 'The file "pseudo.osm.pbf" is not found' in out 66 | 67 | pseudo_dir = os.path.dirname(path_to_pseudo_pbf_file) 68 | os.makedirs(pseudo_dir) 69 | f = open(path_to_pseudo_pbf_file, 'w+') 70 | 71 | with pytest.raises(Exception) as e: 72 | assert os.path.exists(path_to_pseudo_pbf_file) 73 | remove_osm_file(path_to_pseudo_pbf_file) 74 | assert "Failed." in str(e.value) 75 | 76 | f.close() 77 | 78 | remove_osm_file(path_to_pseudo_pbf_file, verbose=True) 79 | out, _ = capfd.readouterr() 80 | assert "Deleting" in out and path_to_pseudo_pbf_file in out 81 | assert not os.path.exists(path_to_pseudo_pbf_file) 82 | 83 | remove_osm_file(pseudo_dir, verbose=True) 84 | assert "Deleting" in out and pseudo_dir in out 85 | assert not os.path.exists(pseudo_dir) 86 | 87 | 88 | if __name__ == '__main__': 89 | pytest.main() 90 | --------------------------------------------------------------------------------