├── .gitignore
├── .readthedocs.yml
├── CHANGELOG.md
├── LICENSE
├── MANIFEST.in
├── README.md
├── docs
├── Makefile
├── make.bat
└── source
│ ├── _images
│ ├── GeometryCollection.png
│ ├── GeometryCollection.svg
│ ├── LineString.png
│ ├── LineString.svg
│ ├── MultiLineString.png
│ ├── MultiLineString.svg
│ ├── MultiPolygon.png
│ ├── MultiPolygon.svg
│ ├── Point.png
│ ├── Point.svg
│ ├── pbf_db_example.png
│ ├── pbf_schemas_example.png
│ ├── pbf_schemas_example_2.png
│ └── pbf_table_example.png
│ ├── _static
│ ├── copy_button.css
│ ├── prompt_button.js
│ └── rtd_overrides.css
│ ├── _templates
│ ├── base.rst
│ ├── breadcrumbs.html
│ ├── class.rst
│ ├── class2.rst
│ ├── function.rst
│ ├── latex.tex_t
│ ├── longtable.tex_t
│ └── module.rst
│ ├── acknowledgement.rst
│ ├── conf.py
│ ├── contributors.rst
│ ├── downloader.rst
│ ├── errors.rst
│ ├── index.rst
│ ├── installation.rst
│ ├── introduction.rst
│ ├── ios.rst
│ ├── latexindex.rst
│ ├── license.rst
│ ├── modules.rst
│ ├── quick-start.rst
│ ├── reader.rst
│ ├── rtd-req.txt
│ └── utils.rst
├── pydriosm
├── __init__.py
├── _updater.py
├── data
│ ├── bbbike_cities.pkl
│ ├── bbbike_cities_coordinates.pkl
│ ├── bbbike_downloads_catalogue.pkl
│ ├── bbbike_index_of_subregions.pkl
│ ├── bbbike_subregion_names.pkl
│ ├── geofabrik_continent_tables.pkl
│ ├── geofabrik_downloads_catalogue.pkl
│ ├── geofabrik_index_of_subregions.pkl
│ ├── geofabrik_region-subregion_tier.pkl
│ ├── geofabrik_subregion_names.pkl
│ └── metadata.json
├── downloader
│ ├── __init__.py
│ ├── _downloader.py
│ ├── bbbike.py
│ └── geofabrik.py
├── errors.py
├── ios
│ ├── __init__.py
│ ├── _ios.py
│ ├── bbbike.py
│ ├── geofabrik.py
│ └── utils.py
├── reader
│ ├── __init__.py
│ ├── _reader.py
│ ├── bbbike.py
│ ├── geofabrik.py
│ ├── parser.py
│ └── transformer.py
└── utils.py
├── pyproject.toml
├── requirements.txt
├── setup.cfg
├── setup.py
└── tests
├── __init__.py
├── data
└── rutland
│ ├── other_relations_1.pkl
│ ├── other_relations_2.pkl
│ ├── points_1.pkl
│ ├── points_2.pkl
│ ├── rutland-latest-free.shp.zip
│ └── rutland-latest.osm.pbf
├── test_downlaoder.py
├── test_errors.py
├── test_ios.py
├── test_reader.py
├── test_updater.py
└── test_utils.py
/.gitignore:
--------------------------------------------------------------------------------
1 | # Environments
2 | venv/
3 | .idea/
4 | */vscode/
5 | **/__pycache__/
6 |
7 | # Sphinx documentation
8 | */build/
9 | **/_generated/
10 |
11 | # Jupyter Notebooks
12 | **/.ipynb_checkpoints/
13 |
14 | # Tests and code coverage
15 | .coverage
16 | .pytest_cache/
17 | cov_reports/
18 |
19 | # Distribution / packaging
20 | dist/
21 | *.egg-info/
22 |
23 | # Bespoke items
24 | *[!_]_.py
25 | .pypirc
26 | tutorials/*
27 | cmd.md
28 |
--------------------------------------------------------------------------------
/.readthedocs.yml:
--------------------------------------------------------------------------------
1 | # .readthedocs.yaml
2 | # Read the Docs configuration file
3 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
4 |
5 | # Required
6 | version: 2
7 |
8 | # Set the version of Python and other tools
9 | build:
10 | os: ubuntu-22.04
11 | tools:
12 | python: "3.9"
13 |
14 | # Build documentation in the docs/ directory with Sphinx (this is the default documentation type)
15 | sphinx:
16 | builder: html
17 | configuration: docs/source/conf.py
18 | fail_on_warning: true
19 |
20 | # Optionally build the docs in additional formats such as PDF
21 | formats:
22 | - pdf
23 |
24 | # Optionally declare the Python requirements required to build the docs
25 | python:
26 | install:
27 | - requirements: docs/source/rtd-req.txt
28 |
--------------------------------------------------------------------------------
/CHANGELOG.md:
--------------------------------------------------------------------------------
1 | ### Changelog / Release notes
2 |
3 |
4 |
5 | #### **[2.2.0](https://github.com/mikeqfu/pydriosm/releases/tag/2.2.0)**
6 |
7 | (*19 June 2023*)
8 |
9 | ##### **Notable [changes](https://github.com/mikeqfu/pydriosm/compare/2.1.2...2.2.0) since [2.1.2](https://pypi.org/project/pydriosm/2.1.2/):**
10 |
11 | - [GDAL](https://pypi.org/project/GDAL/) is now an optional dependency and will only be required when dealing with PBF data.
12 | - Fixed bugs in the following classes: [BBBikeDownloader](https://github.com/mikeqfu/pydriosm/commit/59c9c8a7381c90cd3e706955a27462697bb2104f) and [PostgresOSM](https://github.com/mikeqfu/pydriosm/commit/496e0fc8cee4c5125cff2529e647394276392287).
13 |
14 | **For more information and detailed specifications, check out [PyDriosm 2.2.0 documentation](https://pydriosm.readthedocs.io/en/2.2.0/).**
15 |
16 |
17 |
18 | #### **[2.1.2](https://github.com/mikeqfu/pydriosm/releases/tag/2.1.2)**
19 |
20 | (*27 February 2023*)
21 |
22 | ##### **Notable [changes](https://github.com/mikeqfu/pydriosm/compare/2.1.1...2.1.2) since [2.1.1](https://pypi.org/project/pydriosm/2.1.1/):**
23 |
24 | - Improved the following methods: [SHPReadParse.merge_layer_shps()](https://github.com/mikeqfu/pydriosm/commit/392d759ae4504c5ebb9d8f1dab6471bf5743a7ff) and [BBBikeDownloader.get_subregion_index()](https://github.com/mikeqfu/pydriosm/commit/57b128310b89e40a12d72a0d72e34597d381ed2b).
25 | - Updated requirements on dependencies to adapt to SQLAlchemy 2.0+.
26 |
27 | **For more information and detailed specifications, check out [PyDriosm 2.1.2 documentation](https://pydriosm.readthedocs.io/en/2.1.2/).**
28 |
29 |
30 |
31 | #### **[2.1.1](https://github.com/mikeqfu/pydriosm/releases/tag/2.1.1)**
32 |
33 | (*9 January 2022*)
34 |
35 | ##### **Notable [changes](https://github.com/mikeqfu/pydriosm/compare/2.1.0...2.1.1) since [2.1.0](https://pypi.org/project/pydriosm/2.1.0/):**
36 |
37 | - Improved the following methods/modules (with bug fixes):
38 | - the methods [.write_to_shapefile()](https://github.com/mikeqfu/pydriosm/commit/57baec84a3d8366f6d8f6f324fbcbdf6e7f67fa6), [.read_layer_shps()](https://github.com/mikeqfu/pydriosm/commit/bec76cb0fc21b152849cbc8cccb2634b04dd59f8#diff-fc8bd4c3f1ee495f89956160ebf3736c1b8f8021e61f3eb14662439f8a781aacL1795-R2028), and [.merge_layer_shps()](https://github.com/mikeqfu/pydriosm/commit/bec76cb0fc21b152849cbc8cccb2634b04dd59f8#diff-fc8bd4c3f1ee495f89956160ebf3736c1b8f8021e61f3eb14662439f8a781aacR2105-R2343) of the class [SHPReadParse](https://github.com/mikeqfu/pydriosm/blob/7d2aa13c30c9df324a431567da69c7813e706c94/pydriosm/reader.py#L1030);
39 | - the modules [downloader](https://github.com/mikeqfu/pydriosm/commit/3404a8ad46b03e921110e695005bd47510c8a4f2#diff-5b569a7b9029a0d3c195aacccacf61b1f151777568a288bbd8703a88f67fc2f3) and [_updater](https://github.com/mikeqfu/pydriosm/commit/3404a8ad46b03e921110e695005bd47510c8a4f2#diff-3464d30b2db28f204143ad7f953d55614ece799bc0ae918cc0ad42c6497b39cf).
40 |
41 | **For more information and detailed specifications, check out [PyDriosm 2.1.1 documentation](https://pydriosm.readthedocs.io/en/2.1.1/).**
42 |
43 |
44 |
45 | #### **[2.1.0](https://github.com/mikeqfu/pydriosm/releases/tag/2.1.0)**
46 |
47 | (*20 November 2022*)
48 |
49 | ***Note that this release is a highly modified version and not compatible with any previous versions.***
50 |
51 | ##### **Notable [changes](https://github.com/mikeqfu/pydriosm/compare/2.0.3...2.1.0) since [2.0.3](https://pypi.org/project/pydriosm/2.0.3/):**
52 |
53 | - Made major modifications and sweeping changes to the modules: [downloader](https://github.com/mikeqfu/pydriosm/commit/2761bc7f3cf265ca9621dc10f46ef6dcbcedf263), [reader](https://github.com/mikeqfu/pydriosm/commit/2e4befe4ea7847cba889aa4f983355db717e59e4), [ios](https://github.com/mikeqfu/pydriosm/commit/e508b11cf121d25356a02975481c4aaabf4ada56) and [utils](https://github.com/mikeqfu/pydriosm/commit/c69fe4f5a863eda4f925904afdf2daa2b6390c60).
54 | - Removed the module [settings](https://github.com/mikeqfu/pydriosm/commit/2916dc938a5890b0a19cd4431fd00f9292c7ec65).
55 | - Replaced the module [updater](https://github.com/mikeqfu/pydriosm/commit/ab6ec0ec4689bc719716ba36f9259834bb269a94) with [_updater](https://github.com/mikeqfu/pydriosm/commit/159ba27ff3410ff53ca4210409da02d03cdc2b7e).
56 | - Added a new module [errors](https://github.com/mikeqfu/pydriosm/commit/d9f60388bd085c375873bfc2f8cc395a6f111de3).
57 |
58 | **For more information and detailed specifications, check out [PyDriosm 2.1.0 documentation](https://pydriosm.readthedocs.io/en/2.1.0/).**
59 |
60 |
61 |
62 | #### **[2.0.3](https://github.com/mikeqfu/pydriosm/releases/tag/2.0.3)**
63 |
64 | (*25 April 2021*)
65 |
66 | ##### **Notable [changes](https://github.com/mikeqfu/pydriosm/compare/2.0.2...2.0.3) since [2.0.2](https://pypi.org/project/pydriosm/2.0.2/):**
67 |
68 | - Renamed the function [~~get_default_shp_crs()~~](https://github.com/mikeqfu/pydriosm/commit/5786c620fee89fa2a0db4c8329f7d9cabf7ea81d#diff-fc8bd4c3f1ee495f89956160ebf3736c1b8f8021e61f3eb14662439f8a781aacL694) to [get_epsg4326_wgs84_crs_ref()](https://github.com/mikeqfu/pydriosm/commit/5786c620fee89fa2a0db4c8329f7d9cabf7ea81d#diff-fc8bd4c3f1ee495f89956160ebf3736c1b8f8021e61f3eb14662439f8a781aacR733) in the module [reader](https://github.com/mikeqfu/pydriosm/blob/924f943de08a03595b3604f71591a0c34952a54e/pydriosm/reader.py).
69 | - Removed the function [get_osm_geom_object_dict()](https://github.com/mikeqfu/pydriosm/blob/d7cb423ae30dc3443139fc6063ea3ce24ed7afd9/pydriosm/utils.py#L145-L167) from the module [utils](https://github.com/mikeqfu/pydriosm/blob/924f943de08a03595b3604f71591a0c34952a54e/pydriosm/utils.py)
70 | - Changed the default package for reading/writing shapefiles from [GeoPandas](https://pypi.org/project/geopandas/) to [PyShp](https://pypi.org/project/pyshp/) (Note that [GeoPandas](https://pypi.org/project/geopandas/) would not be required for installing [PyDriosm 2.0.3](https://pypi.org/project/pydriosm/2.0.3/)+ but would still be reserved as an alternative option if available.
71 | - Modified the default download directories.
72 | - Improved the class [PostgresOSM](https://github.com/mikeqfu/pydriosm/commit/90587bb0ab7bd26d3597481d463b280cdaf1a728) in the module [ios](https://github.com/mikeqfu/pydriosm/blob/924f943de08a03595b3604f71591a0c34952a54e/pydriosm/ios.py) and the module [downloader](https://github.com/mikeqfu/pydriosm/commit/52f76723a84cd822fc002f89bf92a744cbd88141) with bug fixes.
73 | - Added the following new functions:
74 | - [get_epsg4326_wgs84_prj_ref()](https://github.com/mikeqfu/pydriosm/commit/5786c620fee89fa2a0db4c8329f7d9cabf7ea81d#diff-fc8bd4c3f1ee495f89956160ebf3736c1b8f8021e61f3eb14662439f8a781aacR765-R791), [specify_pyshp_fields()](https://github.com/mikeqfu/pydriosm/commit/5786c620fee89fa2a0db4c8329f7d9cabf7ea81d#diff-fc8bd4c3f1ee495f89956160ebf3736c1b8f8021e61f3eb14662439f8a781aacR794-R834) and [write_to_shapefile()](https://github.com/mikeqfu/pydriosm/commit/5786c620fee89fa2a0db4c8329f7d9cabf7ea81d#diff-fc8bd4c3f1ee495f89956160ebf3736c1b8f8021e61f3eb14662439f8a781aacR837-R917) to the module [reader](https://github.com/mikeqfu/pydriosm/blob/924f943de08a03595b3604f71591a0c34952a54e/pydriosm/reader.py);
75 | - [shp_shape_types_dict()](https://github.com/mikeqfu/pydriosm/commit/1a82b3d96383500898af13f6a49023f6c88f4c06#diff-262651b10b835e2d78c1c6d4157b36f97721b7a10a13f197715ee984266c3882R147-R172) and [shp_shape_types_geom_dict()](https://github.com/mikeqfu/pydriosm/commit/1a82b3d96383500898af13f6a49023f6c88f4c06#diff-262651b10b835e2d78c1c6d4157b36f97721b7a10a13f197715ee984266c3882R175-R192) to the module [utils](https://github.com/mikeqfu/pydriosm/blob/924f943de08a03595b3604f71591a0c34952a54e/pydriosm/utils.py).
76 |
77 | **For more information and detailed specifications, check out [PyDriosm 2.0.3 documentation](https://pydriosm.readthedocs.io/en/2.0.3/).**
78 |
79 |
80 |
81 | #### **[2.0.2](https://github.com/mikeqfu/pydriosm/releases/tag/2.0.2)**
82 |
83 | (*24 November 2020*)
84 |
85 | ##### **Notable [changes](https://github.com/mikeqfu/pydriosm/compare/2.0.1...2.0.2) since [2.0.1](https://pypi.org/project/pydriosm/2.0.1/):**
86 |
87 | - Added [a new parameter](https://github.com/mikeqfu/pydriosm/commit/3b4d8c3c58f40f4a405586594fa57524a8e825e8) `max_tmpfile_size` for setting the maximum size of in-memory temporary file while instantiating the classes [GeofabrikReader](https://github.com/mikeqfu/pydriosm/commit/3b4d8c3c58f40f4a405586594fa57524a8e825e8#diff-fc8bd4c3f1ee495f89956160ebf3736c1b8f8021e61f3eb14662439f8a781aacR1259) and [BBBikeReader](https://github.com/mikeqfu/pydriosm/commit/3b4d8c3c58f40f4a405586594fa57524a8e825e8#diff-fc8bd4c3f1ee495f89956160ebf3736c1b8f8021e61f3eb14662439f8a781aacR2123) for reading OSM data.
88 | - Added a new function [validate_shp_layer_names()](https://pydriosm.readthedocs.io/en/2.0.2/_generated/pydriosm.utils.validate_shp_layer_names.html) to the module [utils](https://pydriosm.readthedocs.io/en/2.0.2/utils.html).
89 | - Optimized import statements for all modules.
90 |
91 | **For more information and detailed specifications, check out [PyDriosm 2.0.2 documentation](https://pydriosm.readthedocs.io/en/2.0.2/).**
92 |
93 |
94 |
95 | #### **[2.0.1](https://github.com/mikeqfu/pydriosm/releases/tag/2.0.1)**
96 |
97 | (*19 November 2020*)
98 |
99 | ##### **Notable [changes](https://github.com/mikeqfu/pydriosm/compare/2.0.0...2.0.1) since [2.0.0](https://pypi.org/project/pydriosm/2.0.0/):**
100 |
101 | - Optimized import statements for the modules [downloader](https://github.com/mikeqfu/pydriosm/commit/a4d000a9f0e435e283e15c0a0db45049335e286c) and [reader](https://github.com/mikeqfu/pydriosm/commit/09ff2fc65986105566ee923b96284a78f503b3ba).
102 |
103 |
104 |
105 | #### **[2.0.0](https://github.com/mikeqfu/pydriosm/releases/tag/2.0.0)**
106 |
107 | (*19 November 2020*)
108 |
109 | This release introduces a highly modified version of the predecessors tagged "1.0.x". Note that this new version is not compatible with any previous versions.
110 |
111 | ##### **Notable [changes](https://github.com/mikeqfu/pydriosm/compare/1.0.20...2.0.0) since [1.0.20](https://pypi.org/project/pydriosm/1.0.20/):**
112 |
113 | - Featured with the following three new modules:
114 | - [**downloader**](https://github.com/mikeqfu/pydriosm/blob/941e9f5b45a0a356eba5a0281307f19807955357/pydriosm/downloader.py), modified from the former [download_GeoFabrik](https://github.com/mikeqfu/pydriosm/blob/371dbce63886cf22f8484337ed5ced826acfcf05/pydriosm/download_GeoFabrik.py) and [download_BBBike](https://github.com/mikeqfu/pydriosm/blob/371dbce63886cf22f8484337ed5ced826acfcf05/pydriosm/download_BBBike.py), for downloading data;
115 | - [**reader**](https://github.com/mikeqfu/pydriosm/blob/941e9f5b45a0a356eba5a0281307f19807955357/pydriosm/reader.py), modified from the former [read_GeoFabrik](https://github.com/mikeqfu/pydriosm/blob/371dbce63886cf22f8484337ed5ced826acfcf05/pydriosm/read_GeoFabrik.py), for reading the data;
116 | - [**ios**](https://github.com/mikeqfu/pydriosm/blob/941e9f5b45a0a356eba5a0281307f19807955357/pydriosm/ios.py), modified from the former [osm_psql](https://github.com/mikeqfu/pydriosm/blob/371dbce63886cf22f8484337ed5ced826acfcf05/pydriosm/osm_psql.py) and [dump_GeoFabrik](https://github.com/mikeqfu/pydriosm/blob/371dbce63886cf22f8484337ed5ced826acfcf05/pydriosm/dump_GeoFabrik.py), for PostgreSQL-based I/O and storage of the data.
117 | - Renamed the rest of the modules, fixed known bugs and added a number of new functions/classes.
118 |
119 |
120 |
121 | #### **[1.0.20](https://github.com/mikeqfu/pydriosm/releases/tag/1.0.20)**
122 |
123 | (*27 January 2020*)
124 |
125 | *Note that [1.0.19](https://pypi.org/project/pydriosm/1.0.19/) had been removed from [GitHub Releases](https://github.com/mikeqfu/pydriosm/releases).*
126 |
127 | ##### **Notable [changes](https://github.com/mikeqfu/pydriosm/compare/1.0.18...1.0.20) since [1.0.18](https://pypi.org/project/pydriosm/1.0.18/):**
128 |
129 | - Removed the function [split_list()](https://github.com/mikeqfu/pydriosm/commit/e3399e8ac602332aa15ddcfb23cece572906d7f4#diff-262651b10b835e2d78c1c6d4157b36f97721b7a10a13f197715ee984266c3882L111-L120) from the module [utils](https://github.com/mikeqfu/pydriosm/blob/e3399e8ac602332aa15ddcfb23cece572906d7f4/pydriosm/utils.py).
130 | - Improved the following class and functions with bug fixes:
131 | - [OSM](https://github.com/mikeqfu/pydriosm/commit/68f1edfd77ab8a9cc78cccf5197b245edf91dd19) in the module [osm_psql](https://github.com/mikeqfu/pydriosm/blob/68f1edfd77ab8a9cc78cccf5197b245edf91dd19/pydriosm/osm_psql.py);
132 | - [regulate_input_subregion_name()](https://github.com/mikeqfu/pydriosm/commit/57511fdc6948b9eb86eb07b99f4b15e1f8161dc9) in [download_GeoFabrik](https://github.com/mikeqfu/pydriosm/blob/57511fdc6948b9eb86eb07b99f4b15e1f8161dc9/pydriosm/download_GeoFabrik.py);
133 | - [read_pbf()](https://github.com/mikeqfu/pydriosm/commit/a3384eea4a628a7b2e75b19d5e9976e7172ece99#diff-c8b9e0cb8aea477d560c1f28ff9d49c58879751c45aea29eb89176cecb41ac0cL512-R602) and [read_pbf()](https://github.com/mikeqfu/pydriosm/commit/a3384eea4a628a7b2e75b19d5e9976e7172ece99#diff-c8b9e0cb8aea477d560c1f28ff9d49c58879751c45aea29eb89176cecb41ac0cL607-R695) in the module [read_GeoFabrik](https://github.com/mikeqfu/pydriosm/blob/a3384eea4a628a7b2e75b19d5e9976e7172ece99/pydriosm/read_GeoFabrik.py);
134 | - [psql_osm_pbf_data_extracts()](https://github.com/mikeqfu/pydriosm/commit/f6b0ef15bde37dd9dc65864003cb36c49b671aec) in [dump_GeoFabrik](https://github.com/mikeqfu/pydriosm/blob/f6b0ef15bde37dd9dc65864003cb36c49b671aec/pydriosm/dump_GeoFabrik.py).
135 |
136 |
137 |
138 | #### **[1.0.18](https://github.com/mikeqfu/pydriosm/releases/tag/1.0.18)**
139 |
140 | (*9 January 2020*)
141 |
142 | ##### **Notable [changes](https://github.com/mikeqfu/pydriosm/compare/1.0.17...1.0.18) since [1.0.17](https://pypi.org/project/pydriosm/1.0.17/):**
143 |
144 | - Integrated the function [collect_bbbike_subregion_download_catalogue()](https://github.com/mikeqfu/pydriosm/commit/92df65fdde05b554b732222942796eb9292e0677#diff-1adc77ee4baedd4f5bf14bb36545835023a25c9f5df660bb5fa33b1082c33688L94) into [collect_bbbike_download_catalogue()](https://github.com/mikeqfu/pydriosm/commit/92df65fdde05b554b732222942796eb9292e0677) in the module [download_BBBike](https://github.com/mikeqfu/pydriosm/blob/92df65fdde05b554b732222942796eb9292e0677/pydriosm/download_BBBike.py).
145 | - Modified the module [download_GeoFabrik](https://github.com/mikeqfu/pydriosm/commit/af47dfb667a721be97ec9ae5eac0000b4571876b#diff-a2d854a6efc7bb0057ad30f933a3cd9ac250a85d4ab74181644827157659939e), allowing it to download data of a deep or shallow set of subregions.
146 | - Improved the following functions with bug fixes:
147 | - [get_subregion_table()](https://github.com/mikeqfu/pydriosm/commit/d3b559f4b14b768eb657d471357ba621b14356a1#diff-a2d854a6efc7bb0057ad30f933a3cd9ac250a85d4ab74181644827157659939eL59-R119) in the module [download_GeoFabrik](https://github.com/mikeqfu/pydriosm/blob/d3b559f4b14b768eb657d471357ba621b14356a1/pydriosm/download_GeoFabrik.py)
148 | - [find_osm_shp_file()](https://github.com/mikeqfu/pydriosm/commit/a2627d8ec1e816f5d349dd9ff272f29a1faa7f2e#diff-c8b9e0cb8aea477d560c1f28ff9d49c58879751c45aea29eb89176cecb41ac0cL24-R58), [merge_multi_shp()](https://github.com/mikeqfu/pydriosm/commit/a2627d8ec1e816f5d349dd9ff272f29a1faa7f2e#diff-c8b9e0cb8aea477d560c1f28ff9d49c58879751c45aea29eb89176cecb41ac0cL139-R246) and [read_pbf()](https://github.com/mikeqfu/pydriosm/commit/b33b6296ede78ebff0af4753007cc3e22b691835#diff-c8b9e0cb8aea477d560c1f28ff9d49c58879751c45aea29eb89176cecb41ac0cL511-R603) in the module [read_GeoFabrik](https://github.com/mikeqfu/pydriosm/blob/b33b6296ede78ebff0af4753007cc3e22b691835/pydriosm/read_GeoFabrik.py)
149 | - Added a new module [update](https://github.com/mikeqfu/pydriosm/commit/2f96a487e2fce263772853b5a50fe7037d443697).
150 | - Added default parameters for PostgreSQL database connection.
151 |
152 |
153 |
154 | #### **[1.0.17](https://github.com/mikeqfu/pydriosm/releases/tag/1.0.17)**
155 |
156 | (*29 November 2019*)
157 |
158 | ##### **Notable [changes](https://github.com/mikeqfu/pydriosm/compare/1.0.16...1.0.17) since [1.0.16](https://pypi.org/project/pydriosm/1.0.16/):**
159 |
160 | - Improved the following functions with bug fixes:
161 | - [collect_subregion_info_catalogue()](https://github.com/mikeqfu/pydriosm/commit/8c39e6be675f163221009b4e6c66c4db904c3ccf#diff-a2d854a6efc7bb0057ad30f933a3cd9ac250a85d4ab74181644827157659939eL126-R209) and [get_default_pathname()](https://github.com/mikeqfu/pydriosm/commit/8c39e6be675f163221009b4e6c66c4db904c3ccf#diff-a2d854a6efc7bb0057ad30f933a3cd9ac250a85d4ab74181644827157659939eL455-R511) in the module [download_GeoFabrik](https://github.com/mikeqfu/pydriosm/blob/8c39e6be675f163221009b4e6c66c4db904c3ccf/pydriosm/download_GeoFabrik.py);
162 | - [merge_multi_shp()](https://github.com/mikeqfu/pydriosm/commit/94d075441dfa6ace22eec6c7c24217a2fcc2343b#diff-c8b9e0cb8aea477d560c1f28ff9d49c58879751c45aea29eb89176cecb41ac0cL140-R218), [read_shp_zip()](https://github.com/mikeqfu/pydriosm/commit/94d075441dfa6ace22eec6c7c24217a2fcc2343b#diff-c8b9e0cb8aea477d560c1f28ff9d49c58879751c45aea29eb89176cecb41ac0cL252-R359) and [read_pbf()](https://github.com/mikeqfu/pydriosm/commit/94d075441dfa6ace22eec6c7c24217a2fcc2343b#diff-c8b9e0cb8aea477d560c1f28ff9d49c58879751c45aea29eb89176cecb41ac0cL559-R661) in the module [read_GeoFabrik](https://github.com/mikeqfu/pydriosm/blob/94d075441dfa6ace22eec6c7c24217a2fcc2343b/pydriosm/read_GeoFabrik.py).
163 |
164 |
165 |
166 | #### **[1.0.16](https://github.com/mikeqfu/pydriosm/releases/tag/1.0.16)**
167 |
168 | (*6 October 2019*)
169 |
170 | ##### **Notable [changes](https://github.com/mikeqfu/pydriosm/compare/1.0.15...1.0.16) since [1.0.15](https://pypi.org/project/pydriosm/1.0.15/):**
171 |
172 | - Fixed some known bugs.
173 |
174 |
175 |
176 | #### **[1.0.15](https://github.com/mikeqfu/pydriosm/releases/tag/1.0.15)**
177 |
178 | (*29 August 2019*)
179 |
180 | *Note that [1.0.14](https://pypi.org/project/pydriosm/1.0.14/), [1.0.13](https://pypi.org/project/pydriosm/1.0.13/) and [1.0.12](https://pypi.org/project/pydriosm/1.0.12/) had been removed from [GitHub Releases](https://github.com/mikeqfu/pydriosm/releases).*
181 |
182 | ##### **Notable [changes](https://github.com/mikeqfu/pydriosm/compare/1.0.11...1.0.15) since [1.0.11](https://pypi.org/project/pydriosm/1.0.11/):**
183 |
184 | - Improved the functions: [extract_shp_zip()](https://github.com/mikeqfu/pydriosm/commit/758bcbd4dc48a03b1bb72c161ba8e87f04a80a82#diff-c8b9e0cb8aea477d560c1f28ff9d49c58879751c45aea29eb89176cecb41ac0cL74-R114), [read_shp()](https://github.com/mikeqfu/pydriosm/commit/758bcbd4dc48a03b1bb72c161ba8e87f04a80a82#diff-c8b9e0cb8aea477d560c1f28ff9d49c58879751c45aea29eb89176cecb41ac0cL168-R214) and [read_shp_zip()](https://github.com/mikeqfu/pydriosm/commit/758bcbd4dc48a03b1bb72c161ba8e87f04a80a82#diff-c8b9e0cb8aea477d560c1f28ff9d49c58879751c45aea29eb89176cecb41ac0cL193-R296) with bug fixes in the module [read_GeoFabrik](https://github.com/mikeqfu/pydriosm/blob/758bcbd4dc48a03b1bb72c161ba8e87f04a80a82/pydriosm/read_GeoFabrik.py).
185 | - Added a new method [OSM.db_exists()](https://github.com/mikeqfu/pydriosm/commit/73ff3b2bee1d85947d86bf32421e90dcabb7d47d#diff-cb2783bddce6ef6c0d7479f7e4ada08bdcec39cb0e9d0af83a4d1398b5737491R72-R76), allowing [OSM.create_db()](https://github.com/mikeqfu/pydriosm/commit/73ff3b2bee1d85947d86bf32421e90dcabb7d47d#diff-cb2783bddce6ef6c0d7479f7e4ada08bdcec39cb0e9d0af83a4d1398b5737491L72-R96) to check whether a database exists.
186 | - Updated the [LICENSE](https://github.com/mikeqfu/pydriosm/commit/90d12a5aaa36882115e89e5e9f7672b9058f7cda).
187 |
188 |
189 |
190 | #### **[1.0.11](https://github.com/mikeqfu/pydriosm/releases/tag/1.0.11)**
191 |
192 | (*4 April 2019*)
193 |
194 | *Note that [1.0.10](https://pypi.org/project/pydriosm/1.0.10/) and [1.0.9](https://pypi.org/project/pydriosm/1.0.9/) had been removed from [GitHub Releases](https://github.com/mikeqfu/pydriosm/releases).*
195 |
196 | ##### **Notable [changes](https://github.com/mikeqfu/pydriosm/compare/1.0.8...1.0.11) since [1.0.8](https://pypi.org/project/pydriosm/1.0.8/):**
197 |
198 | - Fixed [a minor bug](https://github.com/mikeqfu/pydriosm/commit/f2b22a5af3e7026c7c0810b1857550249c9fc61a) for creating a default data directory.
199 | - Improved the following functions (with bug fixes):
200 | - [get_default_pathname()](https://github.com/mikeqfu/pydriosm/commit/f2b22a5af3e7026c7c0810b1857550249c9fc61a) in the module [download_GeoFabrik](https://github.com/mikeqfu/pydriosm/blob/42c47d60c1a30c37c80b9757fd4c32e60f053bd3/pydriosm/download_GeoFabrik.py);
201 | - [parse_layer_data()](https://github.com/mikeqfu/pydriosm/commit/ec968392139282e8c66d7d0c477f9e6c5967e56c#diff-c8b9e0cb8aea477d560c1f28ff9d49c58879751c45aea29eb89176cecb41ac0cL297-R375) in the module [read_GeoFabrik](https://github.com/mikeqfu/pydriosm/blob/42c47d60c1a30c37c80b9757fd4c32e60f053bd3/pydriosm/read_GeoFabrik.py);
202 | - [dump_osm_pbf_data_by_layer()](https://github.com/mikeqfu/pydriosm/commit/ec968392139282e8c66d7d0c477f9e6c5967e56c#diff-cb2783bddce6ef6c0d7479f7e4ada08bdcec39cb0e9d0af83a4d1398b5737491L170-R179) in the module [osm_psql](https://github.com/mikeqfu/pydriosm/blob/42c47d60c1a30c37c80b9757fd4c32e60f053bd3/pydriosm/osm_psql.py);
203 | - [psql_osm_pbf_data_extracts()](https://github.com/mikeqfu/pydriosm/commit/ec968392139282e8c66d7d0c477f9e6c5967e56c#diff-06caa7c5b7806a98b9c915f4b9e44a9b7c305ead0c66e31baae94a58370c4615L68-R74) in the module [dump_GeoFabrik](https://github.com/mikeqfu/pydriosm/blob/42c47d60c1a30c37c80b9757fd4c32e60f053bd3/pydriosm/dump_GeoFabrik.py), [with a new parameter](https://github.com/mikeqfu/pydriosm/commit/9846653bb2d08580b972a0dbf10c84b1e8bd9050) `database_name` for specifying a database name when dumping data to a PostgreSQL server.
204 | - Added a function [regulate_table_name()](https://github.com/mikeqfu/pydriosm/commit/4cfdd7ebcb489b7b618f6c6163cad9354c071b77#diff-cb2783bddce6ef6c0d7479f7e4ada08bdcec39cb0e9d0af83a4d1398b5737491R17-R27), which regulates PostgreSQL table names, to the module [osm_psql](https://github.com/mikeqfu/pydriosm/blob/305be3f0996be2aa3f5003c3f96b06466d769f50/pydriosm/osm_psql.py).
205 |
206 |
207 |
208 | #### **[1.0.8](https://github.com/mikeqfu/pydriosm/releases/tag/1.0.8)**
209 |
210 | (*18 Mar 2019*)
211 |
212 | *Note that [1.0.6](https://pypi.org/project/pydriosm/1.0.6/) and [1.0.7](https://pypi.org/project/pydriosm/1.0.7/) had been removed from [GitHub Releases](https://github.com/mikeqfu/pydriosm/releases).*
213 |
214 | ##### **Notable [changes](https://github.com/mikeqfu/pydriosm/compare/1.0.5...1.0.8) since [1.0.5](https://pypi.org/project/pydriosm/1.0.5/):**
215 |
216 | - Made [some major changes](https://github.com/mikeqfu/pydriosm/commit/a415ed5d8b6394342a9ac9fb53bb041a3133fa44) (with potential bug fixes).
217 | - Fixed minor bugs in the following functions:
218 | - [parse_layer_data()](https://github.com/mikeqfu/pydriosm/commit/d266c3e49cf8a0d4e1065e60c5e3a6a657ff9332) and [read_shp_zip()](https://github.com/mikeqfu/pydriosm/commit/613f0a9fb3c70db9094590e9f614f254000369ce) in the module [read_GeoFabrik](https://github.com/mikeqfu/pydriosm/blob/305be3f0996be2aa3f5003c3f96b06466d769f50/pydriosm/read_GeoFabrik.py);
219 | - [retrieve_subregions()](https://github.com/mikeqfu/pydriosm/commit/bfbbbb4fc71108845df8fc1eae5d590a7b693d92) and [psql_subregion_osm_data_extracts()](https://github.com/mikeqfu/pydriosm/commit/f6fada22e192a56bd2d6fc250bdaedf9f6d00041) in the module [dump_GeoFabrik](https://github.com/mikeqfu/pydriosm/blob/305be3f0996be2aa3f5003c3f96b06466d769f50/pydriosm/dump_GeoFabrik.py).
220 | - Added a function [regulate_input_data_dir()](https://github.com/mikeqfu/pydriosm/commit/93792ec3493d0b13237cf24c531353f0e77d8f67) to the module [utils](https://github.com/mikeqfu/pydriosm/blob/305be3f0996be2aa3f5003c3f96b06466d769f50/pydriosm/utils.py).
221 |
222 |
223 |
224 | #### **[1.0.5](https://github.com/mikeqfu/pydriosm/releases/tag/1.0.5)**
225 |
226 | (*11 March 2019*)
227 |
228 | *Note that [1.0.4](https://pypi.org/project/pydriosm/1.0.4/), [1.0.3](https://pypi.org/project/pydriosm/1.0.3/), [1.0.2](https://pypi.org/project/pydriosm/1.0.2/) and [1.0.1](https://pypi.org/project/pydriosm/1.0.1/) had been removed from [GitHub Releases](https://github.com/mikeqfu/pydriosm/releases).*
229 |
230 | ##### **Notable [changes](https://github.com/mikeqfu/pydriosm/compare/1.0.0...1.0.5) since [1.0.0](https://pypi.org/project/pydriosm/1.0.0/):**
231 |
232 | - Integrated the function [read_parsed_osm_pbf()](https://github.com/mikeqfu/pydriosm/blob/243788f02c10fa91024b165819b52e6973fa3b26/pydriosm/read_GeoFabrik.py#L474-L515) into [read_pbf()](https://github.com/mikeqfu/pydriosm/blob/243788f02c10fa91024b165819b52e6973fa3b26/pydriosm/read_GeoFabrik.py#L319-L391) in the module [read_GeoFabrik](https://github.com/mikeqfu/pydriosm/blob/a1fb4ddce3f451e29f79fc1e06b999ab9e4eb0b2/pydriosm/read_GeoFabrik.py).
233 | - Improved the following functions
234 | - [dump_osm_pbf_data()](https://github.com/mikeqfu/pydriosm/commit/cd209d985a3270b90d22501fdc5e3a8e8b142ac4#diff-cb2783bddce6ef6c0d7479f7e4ada08bdcec39cb0e9d0af83a4d1398b5737491L173-R204) in the module [osm_psql](https://github.com/mikeqfu/pydriosm/blob/cd209d985a3270b90d22501fdc5e3a8e8b142ac4/pydriosm/osm_psql.py), with a new parameter `chunk_size` allowing users to parse/read/dump data in a chunk-wise way;
235 | - [psql_subregion_osm_data_extracts()](https://github.com/mikeqfu/pydriosm/blob/9b37bbe76223332b037f12a8fa49d1fdb24c7262/pydriosm/dump_GeoFabrik.py#L45-L152) in the module [dump_GeoFabrik](https://github.com/mikeqfu/pydriosm/blob/4558a89938fa6f28ab105a6ee5d54a95745302e2/pydriosm/dump_GeoFabrik.py);
236 | - [gdal_configurations()](https://github.com/mikeqfu/pydriosm/commit/d85af9a3a37cbb8ffb2280090844dc61bde706f2) in the module [settings](https://github.com/mikeqfu/pydriosm/blob/d85af9a3a37cbb8ffb2280090844dc61bde706f2/pydriosm/settings.py).
237 | - Added new function:
238 | - [retrieve_subregions()](https://github.com/mikeqfu/pydriosm/commit/4558a89938fa6f28ab105a6ee5d54a95745302e2#diff-06caa7c5b7806a98b9c915f4b9e44a9b7c305ead0c66e31baae94a58370c4615R19-R40), which retrieves a list of subregions of a given (sub)region name, to the module [dump_GeoFabrik](https://github.com/mikeqfu/pydriosm/blob/4558a89938fa6f28ab105a6ee5d54a95745302e2/pydriosm/dump_GeoFabrik.py);
239 | - [split_list()](https://github.com/mikeqfu/pydriosm/commit/243788f02c10fa91024b165819b52e6973fa3b26#diff-262651b10b835e2d78c1c6d4157b36f97721b7a10a13f197715ee984266c3882R242-R249) to the module [utils](https://github.com/mikeqfu/pydriosm/blob/243788f02c10fa91024b165819b52e6973fa3b26/pydriosm/utils.py).
240 |
241 |
242 |
243 | #### **[1.0.0](https://github.com/mikeqfu/pydriosm/releases/tag/1.0.0)**
244 |
245 | (*4 March 2019*)
246 |
247 | This is a release of a **brand-new** version.
248 |
249 | *Note that the initial releases (of early versions up to **~~0.2.9~~**) had been permanently deleted.*
--------------------------------------------------------------------------------
/MANIFEST.in:
--------------------------------------------------------------------------------
1 | # Include package data
2 | recursive-include pydriosm/data *
3 |
4 | # Exclude tests
5 | recursive-exclude tests *
6 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # PyDriosm
2 |
3 | [](https://pypi.org/project/pydriosm/)
4 | [](https://docs.python.org/3/)
5 | [](https://pydriosm.readthedocs.io/en/latest/?badge=latest)
6 | [](https://github.com/mikeqfu/pydriosm/blob/master/LICENSE)
7 | [](https://app.codacy.com/gh/mikeqfu/pydriosm/dashboard?utm_source=gh&utm_medium=referral&utm_content=&utm_campaign=Badge_grade)
8 | [](https://zenodo.org/badge/latestdoi/92493726)
9 |
10 | PyDriosm is an open-source tool that provides an effortless way to download and access [OpenStreetMap](https://www.openstreetmap.org/) (OSM) data in popular file formats, such as [shapefile](https://wiki.openstreetmap.org/wiki/Shapefiles) and [protobuf binary format](https://wiki.openstreetmap.org/wiki/PBF_Format) (PBF), which are freely available from [Geofabrik](https://download.geofabrik.de/) and [BBBike](https://download.bbbike.org/). Additionally, the package offers a comprehensive solution for convenient I/O operations and efficient storage capabilities for parsed OSM data within [PostgreSQL](https://www.postgresql.org/) databases. This means that users can easily read from and write to PostgreSQL databases, enabling efficient data manipulation, querying, and other essential tasks. Whether you are a researcher, practitioner, or simply interested in working with OSM data, PyDriosm can be useful and helpful to streamline your workflow and enhance your experience.
11 |
12 | ## Installation
13 |
14 | To install the latest release of PyDriosm from [PyPI](https://pypi.org/project/pydriosm/) via [pip](https://pip.pypa.io/en/stable/cli/pip/):
15 |
16 | ```bash
17 | pip install --upgrade pydriosm
18 | ```
19 |
20 | Please also refer to [Installation](https://pydriosm.readthedocs.io/en/latest/installation.html) for more information.
21 |
22 | ## Quick start
23 |
24 | For a concise guide with practical examples, please check out the [quick-start tutorial](https://pydriosm.readthedocs.io/en/latest/quick-start.html). This tutorial showcases how to utilise PyDriosm for various tasks, such as downloading, parsing, and performing storage I/O operations on OSM data using a PostgreSQL database.
25 |
26 | ## Documentation
27 |
28 | The complete PyDriosm documentation: [[HTML](https://pydriosm.readthedocs.io/en/latest/)\] \[[PDF](https://pydriosm.readthedocs.io/_/downloads/en/latest/pdf/)]
29 |
30 | It is hosted on [ReadTheDocs](https://readthedocs.org/projects/pydriosm/) and provides a wealth of detailed examples.
31 |
32 | ## License
33 |
34 | - PyDriosm is licensed under [GNU General Public License v3.0](https://github.com/mikeqfu/pydriosm/blob/master/LICENSE) or later (GPLv3+).
35 | - The free [OpenStreetMap](https://www.openstreetmap.org/) data, which is used for the development of PyDriosm, is licensed under the [Open Data Commons Open Database License](https://opendatacommons.org/licenses/odbl/) (ODbL) by the [OpenStreetMap Foundation](https://osmfoundation.org/) (OSMF).
36 |
37 | ## Acknowledgement
38 |
39 | The development of PyDriosm, including the example code that demonstrates how to use the package, heavily relies on freely available [OpenStreetMap](https://www.openstreetmap.org/) data. The author would like to express sincere gratitude to all the [OpenStreetMap contributors](https://wiki.openstreetmap.org/wiki/Contributors) for their invaluable contributions in making this data accessible to the community.
40 |
41 | ## Cite as
42 |
43 | Fu, Q. (2020). PyDriosm: an open-source tool for downloading, reading and PostgreSQL-based I/O of OpenStreetMap data. Zenodo. [doi:10.5281/zenodo.4281194](https://doi.org/10.5281/zenodo.4281194)
44 |
45 | ```bibtex
46 | @software{qian_fu_pydriosm_4281194,
47 | author = {Qian Fu},
48 | title = {{PyDriosm: an open-source tool for downloading, reading
49 | and PostgreSQL-based I/O of OpenStreetMap data}},
50 | year = 2020,
51 | publisher = {Zenodo},
52 | doi = {10.5281/zenodo.4718623},
53 | url = {https://doi.org/10.5281/zenodo.4281194}
54 | }
55 | ```
56 |
57 | (Please also refer to the export options from [Zenodo](https://zenodo.org/search?page=1&size=20&q=conceptrecid:4281194&all_versions&sort=-version) to reference the specific version of PyDriosm as appropriate.)
58 |
59 | ## Contributors
60 |
61 |
62 |
63 |
64 |
65 |
66 |  Qian Fu
67 | 🌱
68 | 💻
69 | 🧪
70 | 📖
71 | |
72 |
73 |
--------------------------------------------------------------------------------
/docs/Makefile:
--------------------------------------------------------------------------------
1 | # Minimal makefile for Sphinx documentation
2 | #
3 |
4 | # You can set these variables from the command line, and also
5 | # from the environment for the first two.
6 | SPHINXOPTS ?=
7 | SPHINXBUILD ?= sphinx-build
8 | SOURCEDIR = source
9 | BUILDDIR = build
10 |
11 | # Put it first so that "make" without argument is like "make help".
12 | help:
13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
14 |
15 | .PHONY: help Makefile
16 |
17 | # Catch-all target: route all unknown targets to Sphinx using the new
18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
19 | %: Makefile
20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
21 |
--------------------------------------------------------------------------------
/docs/make.bat:
--------------------------------------------------------------------------------
1 | @ECHO OFF
2 |
3 | pushd %~dp0
4 |
5 | REM Command file for Sphinx documentation
6 |
7 | if "%SPHINXBUILD%" == "" (
8 | set SPHINXBUILD=sphinx-build
9 | )
10 | set SOURCEDIR=source
11 | set BUILDDIR=build
12 |
13 | if "%1" == "" goto help
14 |
15 | %SPHINXBUILD% >NUL 2>NUL
16 | if errorlevel 9009 (
17 | echo.
18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
19 | echo.installed, then set the SPHINXBUILD environment variable to point
20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you
21 | echo.may add the Sphinx directory to PATH.
22 | echo.
23 | echo.If you don't have Sphinx installed, grab it from
24 | echo.http://sphinx-doc.org/
25 | exit /b 1
26 | )
27 |
28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
29 | goto end
30 |
31 | :help
32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
33 |
34 | :end
35 | popd
36 |
--------------------------------------------------------------------------------
/docs/source/_images/GeometryCollection.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/docs/source/_images/GeometryCollection.png
--------------------------------------------------------------------------------
/docs/source/_images/LineString.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/docs/source/_images/LineString.png
--------------------------------------------------------------------------------
/docs/source/_images/MultiLineString.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/docs/source/_images/MultiLineString.png
--------------------------------------------------------------------------------
/docs/source/_images/MultiPolygon.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/docs/source/_images/MultiPolygon.png
--------------------------------------------------------------------------------
/docs/source/_images/Point.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/docs/source/_images/Point.png
--------------------------------------------------------------------------------
/docs/source/_images/pbf_db_example.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/docs/source/_images/pbf_db_example.png
--------------------------------------------------------------------------------
/docs/source/_images/pbf_schemas_example.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/docs/source/_images/pbf_schemas_example.png
--------------------------------------------------------------------------------
/docs/source/_images/pbf_schemas_example_2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/docs/source/_images/pbf_schemas_example_2.png
--------------------------------------------------------------------------------
/docs/source/_images/pbf_table_example.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/docs/source/_images/pbf_table_example.png
--------------------------------------------------------------------------------
/docs/source/_static/copy_button.css:
--------------------------------------------------------------------------------
1 | /* Copy buttons */
2 | button.copybtn {
3 | position: absolute;
4 | display: flex;
5 | top: auto;
6 | bottom: .5em; /* 1.5em; */
7 | right: .2em; /* 2.0em; */
8 | width: 1.7em;
9 | height: 1.7em;
10 | opacity: 0;
11 | transition: opacity 0.3s, border .3s, background-color .3s;
12 | user-select: none;
13 | padding: 0;
14 | border: none; /* #e1e1e1 1px solid; */
15 | outline: none;
16 | border-radius: 0.4em;
17 | background-color: rgb(245, 245, 245);
18 | }
19 |
20 | button.copybtn.success {
21 | border-color: #22863a;
22 | }
23 |
24 | button.copybtn img {
25 | width: 100%;
26 | padding: .2em;
27 | }
28 |
29 | div.highlight {
30 | position: relative;
31 | }
32 |
33 | .highlight:hover button.copybtn {
34 | opacity: 1;
35 | }
36 |
37 | .highlight button.copybtn:hover {
38 | background-color: rgb(235, 235, 235);
39 | }
40 |
41 | .highlight button.copybtn:active {
42 | background-color: rgb(187, 187, 187);
43 | }
44 |
45 | /**
46 | * A minimal CSS-only tooltip copied from:
47 | * https://codepen.io/mildrenben/pen/rVBrpK
48 | *
49 | * To use, write HTML like the following:
50 | *
51 | * Short
52 | */
53 | .o-tooltip--left {
54 | position: relative;
55 | }
56 |
57 | .o-tooltip--left:after {
58 | opacity: 0;
59 | visibility: hidden;
60 | position: absolute;
61 | content: attr(data-tooltip);
62 | padding: .2em;
63 | font-size: .8em;
64 | left: -.2em;
65 | background: grey;
66 | color: white;
67 | white-space: nowrap;
68 | z-index: 2;
69 | border-radius: 2px;
70 | transform: translateX(-102%) translateY(0);
71 | transition: opacity 0.2s cubic-bezier(0.64, 0.09, 0.08, 1), transform 0.2s cubic-bezier(0.64, 0.09, 0.08, 1);
72 | }
73 |
74 | .o-tooltip--left:hover:after {
75 | display: block;
76 | opacity: 1;
77 | visibility: visible;
78 | transform: translateX(-100%) translateY(0);
79 | transition: opacity 0.2s cubic-bezier(0.64, 0.09, 0.08, 1), transform 0.2s cubic-bezier(0.64, 0.09, 0.08, 1);
80 | transition-delay: .5s;
81 | }
82 |
--------------------------------------------------------------------------------
/docs/source/_static/prompt_button.js:
--------------------------------------------------------------------------------
1 | $(document).ready(function() {
2 | /* Add a [>>>] button on the top-right corner of code samples to hide the >>> and ... prompts
3 | * and the output, thus making the code copyable.
4 | * Resource: https://docs.python.org/3/_static/copybutton.js */
5 | let div = $('.highlight-python .highlight,' +
6 | '.highlight-python3 .highlight,' +
7 | '.highlight-pycon .highlight,' +
8 | '.highlight-default .highlight');
9 | let pre = div.find('pre');
10 |
11 | // Get the styles from the current theme
12 | pre.parent().parent().css('position', 'relative');
13 | let hide_text = 'Hide the prompts and output';
14 | let show_text = 'Show the prompts and output';
15 | let border_width = pre.css('border-top-width');
16 | let border_style = pre.css('border-top-style');
17 | let border_color = pre.css('border-top-color');
18 | let button_styles = {
19 | 'cursor':'pointer',
20 | 'position': 'absolute',
21 | 'top': '0',
22 | 'right': '0',
23 | 'border-color': border_color,
24 | 'border-style': border_style,
25 | 'border-width': border_width,
26 | 'color': border_color,
27 | 'text-size': '75%',
28 | 'font-family': 'monospace',
29 | 'padding-left': '0.2em',
30 | 'padding-right': '0.2em',
31 | 'border-radius': '0 3px 0 0'
32 | }
33 |
34 | // Create and add the button to all the code blocks that contain >>>
35 | div.each(function() {
36 | let jthis = $(this);
37 | if (jthis.find('.gp').length > 0) {
38 | let button = $('>>>');
39 | button.css(button_styles)
40 | button.attr('title', hide_text);
41 | button.data('hidden', 'false');
42 | jthis.prepend(button);
43 | }
44 | /* tracebacks (.gt) contain bare text elements that need to be wrapped
45 | * in a span to work with .nextUntil() (see later) */
46 | jthis.find('pre:has(.gt)').contents().filter(function() {
47 | return ((this.nodeType === 3) && (this.data.trim().length > 0));
48 | }).wrap('');
49 | });
50 |
51 | // Define the behavior of the button when it's clicked
52 | $('.promptbutton').click(function(e){
53 | e.preventDefault();
54 | let button = $(this);
55 | if (button.data('hidden') === 'false') {
56 | // hide the code output
57 | button.parent().find('.go, .gp, .gt').hide();
58 | button.next('pre').find('.gt').nextUntil('.gp, .go').css('visibility', 'hidden');
59 | button.css('text-decoration', 'line-through');
60 | button.attr('title', show_text);
61 | button.data('hidden', 'true');
62 | } else {
63 | // show the code output
64 | button.parent().find('.go, .gp, .gt').show();
65 | button.next('pre').find('.gt').nextUntil('.gp, .go').css('visibility', 'visible');
66 | button.css('text-decoration', 'none');
67 | button.attr('title', hide_text);
68 | button.data('hidden', 'false');
69 | }
70 | });
71 | });
72 |
--------------------------------------------------------------------------------
/docs/source/_static/rtd_overrides.css:
--------------------------------------------------------------------------------
1 | @media screen and (min-width: 767px) {
2 |
3 | .wy-table-responsive table td {
4 | /* !important prevents the common CSS stylesheets from overriding
5 | this as on RTD they are loaded after this stylesheet */
6 | white-space: normal !important;
7 | }
8 |
9 | .wy-table-responsive {
10 | overflow: visible !important;
11 | }
12 |
13 | .wy-nav-content {
14 | max-width: 72% !important;
15 | }
16 |
17 | code.literal {
18 | color: #404040 !important;
19 | background-color: #fbfbfb !important;
20 | }
21 | }
--------------------------------------------------------------------------------
/docs/source/_templates/base.rst:
--------------------------------------------------------------------------------
1 | {{ objname | escape | underline}}
2 |
3 | .. currentmodule:: {{ module }}
4 |
5 | .. auto{{ objtype }}:: {{ objname }}
6 |
7 |
--------------------------------------------------------------------------------
/docs/source/_templates/breadcrumbs.html:
--------------------------------------------------------------------------------
1 | {%- extends "sphinx_rtd_theme/breadcrumbs.html" %}
2 |
3 | {% block breadcrumbs_aside %}
4 | {% endblock %}
--------------------------------------------------------------------------------
/docs/source/_templates/class.rst:
--------------------------------------------------------------------------------
1 | {{ objname | escape | underline }}
2 |
3 | .. currentmodule:: {{ module }}.{{ objname }}
4 |
5 | .. autoclass:: {{ module }}.{{ objname }}
6 | {% block attributes %}
7 | {% if attributes %}
8 | .. rubric:: {{ _('Attributes') }}
9 | .. autosummary::
10 | :template: base.rst
11 | :toctree:
12 | {% for item in attributes %}
13 | {%- if (not item.startswith('_')) and (item not in inherited_members) %}
14 | {{ item }}
15 | {%- endif -%}
16 | {%- endfor %}
17 | {% endif %}
18 | {% endblock %}
19 | {% block methods %}
20 | {% if methods %}
21 | .. rubric:: {{ _('Methods') }}
22 | .. autosummary::
23 | :template: base.rst
24 | :toctree:
25 | {% for item in methods %}
26 | {%- if (not item.startswith('_') or item in ['__call__']) and (item not in inherited_members) %}
27 | {{ item }}
28 | {%- endif -%}
29 | {%- endfor %}
30 | {% endif %}
31 | {% endblock %}
32 |
33 |
--------------------------------------------------------------------------------
/docs/source/_templates/class2.rst:
--------------------------------------------------------------------------------
1 | {{ objname | escape | underline }}
2 |
3 | .. currentmodule:: {{ module }}.{{ objname }}
4 |
5 | .. autoclass:: {{ module }}.{{ objname }}
6 | {% block attributes %}
7 | {% if attributes %}
8 | .. autosummary::
9 | :template: base.rst
10 | :toctree:
11 | {% for item in attributes %}
12 | {%- if (not item.startswith('_')) and (item not in inherited_members) %}
13 | {{ item }}
14 | {%- endif -%}
15 | {%- endfor %}
16 | {% endif %}
17 | {% endblock %}
18 | {% block methods %}
19 | {% if methods %}
20 | .. autosummary::
21 | :template: base.rst
22 | :toctree:
23 | {% for item in methods %}
24 | {%- if (not item.startswith('_') or item in ['__call__']) and (item not in inherited_members) %}
25 | {{ item }}
26 | {%- endif -%}
27 | {%- endfor %}
28 | {% endif %}
29 | {% endblock %}
30 |
31 |
--------------------------------------------------------------------------------
/docs/source/_templates/function.rst:
--------------------------------------------------------------------------------
1 | {{ objname | escape | underline }}
2 |
3 | .. currentmodule:: {{module}}
4 |
5 | .. autofunction:: {{objname}}
6 |
7 |
--------------------------------------------------------------------------------
/docs/source/_templates/latex.tex_t:
--------------------------------------------------------------------------------
1 | %% Generated by Sphinx.
2 | \def\sphinxdocclass{<%= docclass %>}
3 | <% if latex_engine == 'lualatex' -%>
4 | \IfFileExists{luatex85.sty}
5 | {\RequirePackage{luatex85}}
6 | {\ifdefined\luatexversion\ifnum\luatexversion>84\relax
7 | \PackageError{sphinx}
8 | {** With this LuaTeX (\the\luatexversion),Sphinx requires luatex85.sty **}
9 | {** Add the LaTeX package luatex85 to your TeX installation, and try again **}
10 | \endinput\fi\fi}
11 | <% endif -%>
12 | \documentclass[<%= papersize %>,<%= pointsize %><%= classoptions %>]{<%= wrapperclass %>}
13 | \usepackage[table]{xcolor}
14 | \definecolor{anti-flashwhite}{rgb}{0.95, 0.95, 0.96}
15 | \ifdefined\pdfpxdimen
16 | \let\sphinxpxdimen\pdfpxdimen\else\newdimen\sphinxpxdimen
17 | \fi \sphinxpxdimen=<%= pxunit %>\relax
18 | <% if use_xindy -%>
19 | %% turn off hyperref patch of \index as sphinx.xdy xindy module takes care of
20 | %% suitable \hyperpage mark-up, working around hyperref-xindy incompatibility
21 | \PassOptionsToPackage{hyperindex=false}{hyperref}
22 | <% endif -%>
23 | <%= passoptionstopackages %>
24 | \PassOptionsToPackage{warn}{textcomp}
25 | <%= inputenc %>
26 | <%= utf8extra %>
27 | <%= cmappkg %>
28 | <%= fontenc %>
29 | <%= amsmath %>
30 | <%= multilingual %>
31 | <%= substitutefont %>
32 | <%= textcyrillic %>
33 | <%= fontpkg %>
34 | <%= textgreek %>
35 | <%= fncychap %>
36 | \usepackage<%= sphinxpkgoptions %>{sphinx}
37 | <%= sphinxsetup %>
38 | <%= fvset %>
39 | <%= geometry %>
40 | <%= extrapackages %>
41 |
42 | <%- for name, option in packages %>
43 | <%- if option %>
44 | \usepackage[<%= option %>]{<%= name %>}
45 | <%- else %>
46 | \usepackage{<%= name %>}
47 | <%- endif %>
48 | <%- endfor %>
49 |
50 | <%= hyperref %>
51 | <%- for name, option in packages_after_hyperref %>
52 | <%- if option %>
53 | \usepackage[<%= option %>]{<%= name %>}
54 | <%- else %>
55 | \usepackage{<%= name %>}
56 | <%- endif %>
57 | <%- endfor %>
58 |
59 | <%= contentsname %>
60 | \usepackage{sphinxmessages}
61 | <%= tocdepth %>
62 | <%= secnumdepth %>
63 | <%= preamble %>
64 |
65 | \title{<%= title %>}
66 | \date{<%= date %>}
67 | \release{<%= release %>}
68 | \author{<%= author %>}
69 | <%- if logofilename %>
70 | \newcommand{\sphinxlogo}{\sphinxincludegraphics{<%= logofilename %>}\par}
71 | <%- else %>
72 | \newcommand{\sphinxlogo}{\vbox{}}
73 | <%- endif %>
74 | <%- if releasename or release %>
75 | \renewcommand{\releasename}{<%= releasename or _('Release') | e %>}
76 | <%- else %>
77 | \renewcommand{\releasename}{}
78 | <%- endif %>
79 | <%= makeindex %>
80 | \begin{document}
81 | <%= shorthandoff %>
82 | \pagestyle{empty}
83 | <%= maketitle %>
84 | \pagestyle{plain}
85 | <%= tableofcontents %>
86 | \pagestyle{normal}
87 | <%= body %>
88 | <%= atendofbody %>
89 | <%= indices %>
90 | \renewcommand{\indexname}{<%= _('Index') | e %>}
91 | <%= printindex %>
92 | \end{document}
93 |
--------------------------------------------------------------------------------
/docs/source/_templates/longtable.tex_t:
--------------------------------------------------------------------------------
1 | \begin{savenotes}\sphinxatlongtablestart\begin{longtable}
2 | <%- if table.align in ('center', 'default') -%>
3 | [c]
4 | <%- elif table.align == 'left' -%>
5 | [l]
6 | <%- elif table.align == 'right' -%>
7 | [r]
8 | <%- endif -%>
9 | <%= table.get_colspec() %> % {\X{6}{9}\X{3}{9}}
10 | <%- if table.caption -%>
11 | \sphinxthelongtablecaptionisattop
12 | \caption*{<%= ''.join(table.caption) %>\strut}<%= labels %>\\*[\sphinxlongtablecapskipadjust]
13 | \hline
14 | <% elif labels -%>
15 | \hline\noalign{\phantomsection<%= labels %>}%
16 | <% else -%>
17 | \hline
18 | <% endif -%>
19 | <%= ''.join(table.header) %>
20 | \endfirsthead
21 |
22 | \multicolumn{<%= table.colcount %>}{l}%
23 | {\makebox[0pt][l]{\sphinxtablecontinued{<%= _('(continued from previous page)') %>}}}\\
24 | \hline
25 | \endhead
26 |
27 | \hline
28 | \multicolumn{<%= table.colcount %>}{r}{\makebox[0pt][r]{\sphinxtablecontinued{<%= _('(continues on next page)') %>}}}\\
29 | \endfoot
30 |
31 | \endlastfoot
32 | <%= ''.join(table.body) %>
33 | \end{longtable}\sphinxatlongtableend\end{savenotes}
34 |
--------------------------------------------------------------------------------
/docs/source/_templates/module.rst:
--------------------------------------------------------------------------------
1 | {{ module | escape | underline }}
2 |
3 | .. currentmodule:: {{ fullname }}
4 |
5 | .. automodule:: {{ fullname }}
6 |
7 | {% if functions %}
8 | .. rubric:: Functions
9 | .. autosummary::
10 | :template: function.rst
11 | :toctree: .
12 | {% for function in functions %}
13 | {{ function }}
14 | {% endfor %}
15 | {% endif %}
16 | {% if classes %}
17 | .. rubric:: Classes
18 | .. autosummary::
19 | :template: class.rst
20 | :toctree: .
21 | {% for class in classes %}
22 | {{ class }}
23 | {% endfor %}
24 | {% endif %}
25 |
26 |
--------------------------------------------------------------------------------
/docs/source/acknowledgement.rst:
--------------------------------------------------------------------------------
1 | ===============
2 | Acknowledgement
3 | ===============
4 |
5 | The development of `pydriosm `_, including the example code that demonstrates how to use the package, heavily relies on freely available `OpenStreetMap `_ data. The author would like to express sincere gratitude to all the `OpenStreetMap contributors `_ for their invaluable contributions in making this data accessible to the community.
6 |
--------------------------------------------------------------------------------
/docs/source/conf.py:
--------------------------------------------------------------------------------
1 | """
2 | Configuration file for the Sphinx documentation builder.
3 | """
4 |
5 | # == Path setup ====================================================================================
6 | import os
7 | import sys
8 |
9 | # If the directory is relative to the documentation root, use os.path.abspath to make it absolute:
10 | sys.path.insert(0, os.path.abspath('../..'))
11 | sys.path.insert(0, os.path.abspath('../../pydriosm'))
12 | sys.path.insert(0, os.path.abspath('../../pydriosm/data'))
13 |
14 | # == Project information ===========================================================================
15 | from pydriosm import __affiliation__, __author__, __copyright__, __description__, __pkgname__, \
16 | __project__, __version__, __first_release_date__
17 |
18 | # General information about the project:
19 | project = __project__
20 | copyright = __copyright__
21 |
22 | # The version info for the project:
23 | version = __version__ # The short X.Y.Z version
24 | release = version # The full version, including alpha/beta/rc tags
25 |
26 | # == General configuration =========================================================================
27 | extensions = [ # Sphinx extension module names, which can be named 'sphinx.ext.*' or custom ones:
28 | 'sphinx.ext.autodoc',
29 | 'sphinx.ext.autosummary',
30 | 'sphinx.ext.autosectionlabel',
31 | 'sphinx.ext.extlinks',
32 | 'sphinx.ext.intersphinx',
33 | 'sphinx.ext.inheritance_diagram',
34 | 'sphinx.ext.githubpages',
35 | 'sphinx.ext.todo',
36 | 'sphinx_rtd_theme',
37 | 'sphinx.ext.linkcode',
38 | 'sphinx.ext.doctest',
39 | 'sphinx_copybutton',
40 | ]
41 |
42 |
43 | def linkcode_resolve(domain, info):
44 | """
45 | Determine the URL corresponding to Python object.
46 |
47 | (Adapted from https://github.com/pandas-dev/pandas/blob/main/doc/source/conf.py)
48 | """
49 |
50 | import inspect
51 | import warnings
52 |
53 | import pydriosm
54 |
55 | if domain != 'py' or not info['module']:
56 | return None
57 |
58 | module_name, full_name = info['module'], info['fullname']
59 |
60 | sub_module_name = sys.modules.get(module_name)
61 | if sub_module_name is None:
62 | return None
63 |
64 | obj = sub_module_name
65 | for part in full_name.split('.'):
66 | try:
67 | with warnings.catch_warnings():
68 | # Accessing deprecated objects will generate noisy warnings
69 | warnings.simplefilter('ignore', FutureWarning)
70 | obj = getattr(obj, part)
71 | except AttributeError:
72 | return None
73 |
74 | try:
75 | fn = inspect.getsourcefile(inspect.unwrap(obj))
76 | except TypeError:
77 | try: # property
78 | fn = inspect.getsourcefile(inspect.unwrap(obj.fget))
79 | except (AttributeError, TypeError):
80 | fn = None
81 | if not fn:
82 | return None
83 |
84 | source = [0]
85 | try:
86 | source, line_no = inspect.getsourcelines(obj)
87 | except TypeError:
88 | try: # property
89 | source, line_no = inspect.getsourcelines(obj.fget)
90 | except (AttributeError, TypeError):
91 | line_no = None
92 | except OSError:
93 | line_no = None
94 |
95 | if line_no:
96 | line_spec = f"#L{line_no}-L{line_no + len(source) - 1}"
97 | else:
98 | line_spec = ""
99 |
100 | # fn = os.path.relpath(fn, start=os.path.abspath(".."))
101 | fn = os.path.relpath(fn, start=os.path.dirname(pydriosm.__file__))
102 |
103 | # f"https://github.com/mikeqfu/pydriosm/blob/{pydriosm.__version__}/pydriosm/{fn}{line_spec}"
104 | url = f"https://github.com/mikeqfu/pydriosm/blob/master/pydriosm/{fn}{line_spec}"
105 |
106 | return url
107 |
108 |
109 | # Enable to reference numbered figures:
110 | numfig = True
111 | numfig_secnum_depth = 0
112 | numfig_format = {'figure': 'Fig. %s', 'table': 'Table %s', 'code-block': 'Code Block %s'}
113 | numfig_format_caption = {'figure': 'Fig. %s: '}
114 |
115 | # The language for content autogenerated by Sphinx:
116 | language = 'en'
117 |
118 | # Add any paths that contain templates here, relative to this directory:
119 | templates_path = ['_templates']
120 |
121 | # Patterns (relative to source directory) that match files & directories to ignore:
122 | exclude_patterns = ['_build', '../_build', '../build']
123 |
124 | # Whether to scan all found documents for autosummary directives and generate stub pages for each:
125 | autosummary_generate = True
126 |
127 | # The suffix(es) of source filenames (For multiple suffix, a list of string:
128 | source_suffix = '.rst' # e.g. source_suffix = ['.rst', '.md'])
129 |
130 | # The master toctree document:
131 | master_doc = 'index'
132 |
133 | # Content inserted into the main body of an `autoclass` directive:
134 | autoclass_content = 'both' # ['class', 'init']
135 |
136 | # Automatically documented members are sorted by source order ('bysource'):
137 | autodoc_member_order = 'bysource'
138 |
139 | # == Options for HTML and HTMLHelp output ==========================================================
140 | html_theme = 'sphinx_rtd_theme' # The theme to use for HTML and HTML Help pages
141 |
142 | html_theme_options = {
143 | 'collapse_navigation': False,
144 | 'navigation_depth': 3,
145 | }
146 |
147 | # Hide/show source link:
148 | html_copy_source = False
149 | html_show_sourcelink = False
150 |
151 | # The name of the Pygments (syntax highlighting) style to use:
152 | pygments_style = 'sphinx' # or 'default'
153 |
154 | # Paths containing custom static files (e.g. style sheets), relative to this directory:
155 | html_static_path = ['_static']
156 |
157 | # Add custom CSS:
158 | html_css_files = ['rtd_overrides.css', 'copy_button.css']
159 |
160 | # Add custom JavaScript:
161 | html_js_files = ['prompt_button.js']
162 |
163 | # Output file base name for HTML help builder:
164 | htmlhelp_basename = __project__ + 'doc' # Defaults to 'pydoc'
165 |
166 | # == Options for LaTeX output ======================================================================
167 | from pygments.formatters.latex import LatexFormatter
168 | from sphinx.highlighting import PygmentsBridge
169 |
170 |
171 | class CustomLatexFormatter(LatexFormatter):
172 | def __init__(self, **options):
173 | super(CustomLatexFormatter, self).__init__(**options)
174 | self.verboptions = r"formatcom=\footnotesize"
175 |
176 |
177 | PygmentsBridge.latex_formatter = CustomLatexFormatter
178 |
179 | # The LaTeX engine to build the docs
180 | latex_engine = 'pdflatex'
181 |
182 | # Grouping the document tree into LaTeX files
183 | latex_documents = [
184 | ('index', # source start file
185 | __pkgname__ + '.tex', # target name
186 | project + ' Documentation', # title
187 | __author__, # author
188 | 'manual', # document class ['howto', 'manual', or own class]
189 | 1 # toctree only
190 | ),
191 | ]
192 |
193 | affil_dept, affil_sch, affil_univ = __affiliation__.split(', ')
194 |
195 | # Custom title page
196 | latex_maketitle = r'''
197 | \newgeometry{top=1.1in,bottom=1.1in,right=1.0in,left=1.0in}
198 | \pagenumbering{roman}
199 | \makeatletter
200 | \hypertarget{titlepage}{}
201 | \begin{titlepage}
202 | \flushright
203 |
204 | \vspace*{22mm}
205 | \textbf{\Huge {{%s}}}
206 |
207 | \vspace{5mm}
208 | \textit{\Large {{%s}}} \par
209 | \vspace{5mm}
210 | \LARGE \textbf{\textit{{Release %s}}} \par
211 |
212 | \vspace{45mm}
213 | \LARGE \textbf{{%s}} \par
214 | \Large \textit{{%s}} \par
215 | \Large \textit{{%s}} \par
216 | \Large \textit{{%s}} \par
217 |
218 | \vspace{50mm}
219 | \Large {{First release:}} \large \textbf{{%s}} \par
220 | \Large {{Last updated:}} \large \textbf{{\MonthYearFormat\today}} \par
221 |
222 | \vspace{15mm}
223 | \large \textcopyright \space Copyright %s \par
224 |
225 | \end{titlepage}
226 | \restoregeometry
227 | \bookmark[dest=titlepage]{Title}
228 | \makeatother
229 |
230 | \clearpage
231 | \pagenumbering{roman}
232 |
233 | \cleardoublepage
234 | \makeatletter
235 | \hypertarget{tocpage}{}
236 | \tableofcontents
237 | \bookmark[dest=tocpage]{Table of Contents}
238 | \makeatother
239 |
240 | \cleardoublepage
241 | \makeatletter
242 | \hypertarget{lofpage}{}
243 | \listoffigures
244 | \bookmark[dest=lofpage]{List of Figures}
245 | \makeatother
246 |
247 | \clearpage
248 | \pagenumbering{arabic}
249 | ''' % (project,
250 | __description__.rstrip('.'),
251 | release,
252 | __author__,
253 | affil_dept,
254 | affil_sch,
255 | affil_univ,
256 | __first_release_date__,
257 | __copyright__)
258 |
259 | latex_preamble = r'''
260 | \setlength{\headheight}{14pt}
261 | \DeclareUnicodeCharacter{229E}{\ensuremath{\boxplus}}
262 | \setcounter{tocdepth}{2}
263 | \setcounter{secnumdepth}{2}
264 | \usepackage{float,textcomp,textgreek,graphicx,blindtext,color,svg,booktabs,newunicodechar}
265 | \usepackage{datetime}
266 | \newdateformat{MonthYearFormat}{%
267 | \monthname[\THEMONTH] \THEYEAR}
268 | \usepackage[none]{hyphenat}
269 | \usepackage[document]{ragged2e}
270 | \usepackage[utf8]{inputenc}
271 | \usepackage[sc,osf]{mathpazo}
272 | \linespread{1.05}
273 | \renewcommand{\sfdefault}{pplj}
274 | \newunicodechar{≤}{\ensuremath{\leq}}
275 | \newunicodechar{≈}{\ensuremath{\approx}}
276 | \newunicodechar{≥}{\ensuremath{\geq}}
277 | \IfFileExists{zlmtt.sty}
278 | {\usepackage[light,scaled=1.05]{zlmtt}}
279 | {\renewcommand{\ttdefault}{lmtt}}
280 | \let\oldlongtable\longtable
281 | \let\endoldlongtable\endlongtable
282 | \renewenvironment{longtable}
283 | {\rowcolors{1}{anti-flashwhite}{white}\oldlongtable}
284 | {\endoldlongtable}
285 | \usepackage[open]{bookmark}
286 | \bookmarksetup{numbered}
287 |
288 | \addto\captionsenglish{\renewcommand{\contentsname}{Table of Contents}}
289 | '''
290 |
291 | # LaTeX customization:
292 | latex_elements = {
293 | 'papersize': 'a4paper', # The paper size ('letterpaper' or 'a4paper')
294 | 'pointsize': '11pt', # The font size ('10pt', '11pt' or '12pt')
295 | 'pxunit': '0.25bp',
296 |
297 | 'fontpkg': '\\usepackage{amsmath,amsfonts,amssymb,amsthm}',
298 | 'fncychap': '\\usepackage{fncychap}', # '\\usepackage[Bjarne]{fncychap}'
299 |
300 | 'preamble': latex_preamble,
301 |
302 | 'figure_align': 'H', # Latex figure (float) alignment; optional: 'htbp'
303 | 'extraclassoptions': 'openany,oneside',
304 |
305 | 'maketitle': latex_maketitle,
306 |
307 | 'releasename': ' ',
308 | 'tableofcontents': ' ',
309 |
310 | # 'makeindex': ' ',
311 | 'printindex': r'''
312 | \IfFileExists{\jobname.ind}
313 | {\footnotesize\raggedright\printindex}
314 | {\begin{sphinxtheindex}\end{sphinxtheindex}}
315 | ''',
316 |
317 | 'fvset': r'\fvset{fontsize=auto}',
318 |
319 | 'sphinxsetup': r'''
320 | %verbatimwithframe=false,
321 | %verbatimwrapslines=false,
322 | %verbatimhintsturnover=false,
323 | VerbatimColor={HTML}{F5F5F5},
324 | VerbatimBorderColor={HTML}{E0E0E0},
325 | noteBorderColor={HTML}{E0E0E0},
326 | noteborder=1.5pt,
327 | warningBorderColor={HTML}{E0E0E0},
328 | warningborder=1.5pt,
329 | warningBgColor={HTML}{FBFBFB},
330 | hmargin={0.7in,0.7in}, vmargin={0.9in,0.9in},
331 | ''',
332 |
333 | 'passoptionstopackages': r'\PassOptionsToPackage{svgnames}{xcolor}',
334 | }
335 |
336 | # == Options for manual page output ================================================================
337 | man_pages = [ # How to group the document tree into manual pages
338 | ('index', # startdocname
339 | __pkgname__, # name
340 | project + ' Documentation', # description
341 | [__author__], # authors
342 | 1 # section
343 | )
344 | ]
345 |
346 | # == Options for Texinfo output ====================================================================
347 | texinfo_documents = [ # Grouping the document tree into Texinfo files
348 | (master_doc, # source start file
349 | __pkgname__, # target name
350 | project + ' Documentation', # title
351 | __author__, # author
352 | project, # dir menu entry
353 | __description__, # description
354 | 'Data manipulation tools', # category
355 | 1 # toctree only
356 | ),
357 | ]
358 |
--------------------------------------------------------------------------------
/docs/source/contributors.rst:
--------------------------------------------------------------------------------
1 | ============
2 | Contributors
3 | ============
4 |
5 | .. only:: html
6 |
7 | .. raw:: html
8 |
9 |
10 |
11 |
12 |
13 |  Qian Fu
14 | 🌱
15 | 💻
16 | 🧪
17 | 📖
18 | |
19 |
20 |
21 |
22 |
23 | .. only:: latex
24 |
25 | - `Qian Fu `_
26 |
--------------------------------------------------------------------------------
/docs/source/downloader.rst:
--------------------------------------------------------------------------------
1 | downloader
2 | ==========
3 |
4 | .. py:module:: pydriosm.downloader
5 |
6 | .. automodule:: pydriosm.downloader
7 | :noindex:
8 | :no-members:
9 | :no-undoc-members:
10 | :no-inherited-members:
11 |
12 | Download OSM data
13 | -----------------
14 |
15 | .. autosummary::
16 | :toctree: _generated/
17 | :template: class.rst
18 |
19 | GeofabrikDownloader
20 | BBBikeDownloader
21 |
--------------------------------------------------------------------------------
/docs/source/errors.rst:
--------------------------------------------------------------------------------
1 | errors
2 | ======
3 |
4 | .. py:module:: pydriosm.errors
5 |
6 | .. automodule:: pydriosm.errors
7 | :noindex:
8 | :no-members:
9 | :no-undoc-members:
10 | :no-inherited-members:
11 |
12 | Validation errors
13 | -----------------
14 |
15 | .. autosummary::
16 | :toctree: _generated/
17 | :template: class.rst
18 |
19 | InvalidSubregionNameError
20 | InvalidFileFormatError
21 |
22 | Parse errors
23 | ------------
24 |
25 | .. autosummary::
26 | :toctree: _generated/
27 | :template: class.rst
28 |
29 | OtherTagsReformatError
30 |
--------------------------------------------------------------------------------
/docs/source/index.rst:
--------------------------------------------------------------------------------
1 | ########
2 | PyDriosm
3 | ########
4 |
5 | |PyPI| |Python Version| |Documentation Status| |License| |Codacy Badge| |DOI|
6 |
7 | .. |PyPI| image:: https://img.shields.io/pypi/v/pydriosm
8 | :alt: PyPI - Release
9 | :target: https://pypi.org/project/pydriosm/
10 | .. |Python Version| image:: https://img.shields.io/pypi/pyversions/pydriosm
11 | :alt: PyPI - Python version
12 | :target: https://docs.python.org/3/
13 | .. |Documentation Status| image:: https://readthedocs.org/projects/pydriosm/badge/?version=latest
14 | :alt: ReadTheDocs - Documentation status
15 | :target: https://pydriosm.readthedocs.io/en/latest/?badge=latest
16 | .. |License| image:: https://img.shields.io/pypi/l/pydriosm
17 | :alt: PyPI - License
18 | :target: https://github.com/mikeqfu/pydriosm/blob/master/LICENSE
19 | .. |Codacy Badge| image:: https://app.codacy.com/project/badge/Grade/b411ce89cbc445f58377a5799646d4cb
20 | :alt: Codacy grade (Code quality)
21 | :target: https://www.codacy.com/gh/mikeqfu/pydriosm/dashboard?utm_source=github.com&utm_medium=referral&utm_content=mikeqfu/pydriosm&utm_campaign=Badge_Grade
22 | .. |DOI| image:: https://zenodo.org/badge/92493726.svg
23 | :alt: Zenodo - DOI
24 | :target: https://zenodo.org/badge/latestdoi/92493726
25 |
26 | | **Author**: Qian Fu
27 | | **Email**: q.fu@bham.ac.uk
28 |
29 | PyDriosm is an open-source tool that provides an effortless way to download and access `OpenStreetMap `_ (OSM) data in popular file formats, such as `shapefile `_ and `protobuf binary format `_ (PBF), which are freely available from `Geofabrik `_ and `BBBike `_. Additionally, the package offers a comprehensive solution for convenient I/O operations and efficient storage capabilities for parsed OSM data within `PostgreSQL `_ databases. This means that users can easily read from and write to PostgreSQL databases, enabling efficient data manipulation, querying, and other essential tasks. Whether you are a researcher, practitioner, or simply interested in working with OSM data, PyDriosm can be useful and helpful to streamline your workflow and enhance your experience.
30 |
31 |
32 | .. toctree::
33 | :maxdepth: 1
34 | :includehidden:
35 | :caption: Documentation
36 |
37 | installation
38 | modules
39 | license
40 | acknowledgement
41 | contributors
42 |
43 | .. toctree::
44 | :maxdepth: 2
45 | :includehidden:
46 | :caption: Tutorial
47 |
48 | quick-start
49 |
50 |
51 | Indices
52 | #######
53 |
54 | * :ref:`genindex`
55 | * :ref:`modindex`
56 | * :ref:`search`
57 |
--------------------------------------------------------------------------------
/docs/source/installation.rst:
--------------------------------------------------------------------------------
1 | ============
2 | Installation
3 | ============
4 |
5 | To install the latest release of PyDriosm from `PyPI`_ via `pip`_:
6 |
7 | .. _`PyPI`: https://pypi.org/project/pydriosm/
8 | .. _`pip`: https://pip.pypa.io/en/stable/cli/pip/
9 |
10 | .. code-block:: console
11 |
12 | pip install --upgrade pydriosm
13 |
14 |
15 | To install the most recent version of PyDriosm hosted on `GitHub`_:
16 |
17 | .. _`GitHub`: https://github.com/mikeqfu/pydriosm
18 |
19 | .. code-block:: console
20 |
21 | pip install --upgrade git+https://github.com/mikeqfu/pydriosm.git
22 |
23 |
24 | .. warning::
25 |
26 | - `Pip`_ may fail to install the dependency package `GDAL`_. In such a circumstance, try instead to `install their .whl files`_, which can be downloaded from the web page of the `archived "unofficial Windows binaries for Python extension packages"`_ (by Christoph Gohlke) or a `mirror site`_ (by Erin Turnbull). For how to install a .whl file, see the answers to this `StackOverflow question`_.
27 |
28 | .. _`GDAL`: https://pypi.org/project/GDAL/
29 | .. _`archived "unofficial Windows binaries for Python extension packages"`: https://www.lfd.uci.edu/~gohlke/pythonlibs/
30 | .. _`mirror site`: http://eturnbull.ca/pythonlibs/
31 | .. _`StackOverflow question`: https://stackoverflow.com/questions/27885397
32 |
33 |
34 | .. note::
35 |
36 | - If using a `virtual environment`_, make sure it is activated.
37 | - It is recommended to add `pip install`_ the option ``--upgrade`` (or ``-U``) to ensure that you are getting the latest stable release of the package.
38 | - Non-essential dependencies (e.g. `GeoPandas`_) of PyDriosm are not enforced to be installed along with the installation of the package. This is intended to optimise the installation requirements. If a `ModuleNotFoundError`_ or an `ImportError`_ pops out when importing/running a function or a method, first try to install the module(s)/package(s) mentioned in the error message, and then try to import/run the function or method again.
39 | - For more general instructions on the installation of Python packages, please refer to the official guide of `Installing Packages`_.
40 |
41 | .. _`virtual environment`: https://packaging.python.org/glossary/#term-Virtual-Environment
42 | .. _`pip install`: https://pip.pypa.io/en/stable/cli/pip_install/
43 | .. _`ModuleNotFoundError`: https://docs.python.org/3/library/exceptions.html#ModuleNotFoundError
44 | .. _`ImportError`: https://docs.python.org/3/library/exceptions.html#ImportError
45 | .. _`GeoPandas`: https://geopandas.org/en/stable/getting_started/install.html#installing-with-pip
46 | .. _`install their .whl files`: https://stackoverflow.com/a/27909082/4981844
47 | .. _`Installing Packages`: https://packaging.python.org/tutorials/installing-packages/
48 |
49 |
50 | To check whether PyDriosm has been correctly installed, try to import the package via an interpreter shell:
51 |
52 | .. code-block:: python
53 | :name: cmd current version
54 |
55 | >>> import pydriosm
56 |
57 | >>> pydriosm.__version__ # Check the latest version
58 |
59 | .. parsed-literal::
60 | The latest version is: |version|
--------------------------------------------------------------------------------
/docs/source/introduction.rst:
--------------------------------------------------------------------------------
1 | :orphan:
2 |
3 | ==============
4 | About PyDriosm
5 | ==============
6 |
7 | PyDriosm is an open-source tool that provides an effortless way to download and access `OpenStreetMap`_ (OSM) data in popular file formats, such as `shapefile`_ and `protobuf binary format`_ (PBF), which are freely available from `Geofabrik`_ and `BBBike`_. Additionally, the package offers a comprehensive solution for convenient I/O operations and efficient storage capabilities for parsed OSM data within `PostgreSQL`_ databases. This means that users can easily read from and write to PostgreSQL databases, enabling efficient data manipulation, querying, and other essential tasks. Whether you are a researcher, practitioner, or simply interested in working with OSM data, PyDriosm is a valuable tool to streamline your workflow and enhance your experience.
8 |
9 | .. _`OpenStreetMap`: https://www.openstreetmap.org/
10 | .. _`shapefile`: https://wiki.openstreetmap.org/wiki/Shapefiles
11 | .. _`protobuf binary format`: https://wiki.openstreetmap.org/wiki/PBF_Format
12 | .. _`Geofabrik`: https://download.geofabrik.de/
13 | .. _`BBBike`: https://download.bbbike.org/
14 | .. _`PostgreSQL`: https://www.postgresql.org/
15 |
--------------------------------------------------------------------------------
/docs/source/ios.rst:
--------------------------------------------------------------------------------
1 | ios
2 | ===
3 |
4 | .. py:module:: pydriosm.ios
5 |
6 | .. automodule:: pydriosm.ios
7 | :noindex:
8 | :no-members:
9 | :no-undoc-members:
10 | :no-inherited-members:
11 |
12 | Manage I/O storage of parsed OSM data
13 | -------------------------------------
14 |
15 | .. autosummary::
16 | :toctree: _generated/
17 | :template: class.rst
18 |
19 | PostgresOSM
20 |
21 | Customised alternatives (Optional)
22 | ----------------------------------
23 |
24 | .. autosummary::
25 | :toctree: _generated/
26 | :template: class2.rst
27 |
28 | GeofabrikIOS
29 | BBBikeIOS
30 |
31 | Other utilities
32 | ---------------
33 |
34 | .. py:module:: pydriosm.ios.utils
35 |
36 | .. automodule:: pydriosm.ios.utils
37 | :noindex:
38 | :no-members:
39 | :no-undoc-members:
40 | :no-inherited-members:
41 |
42 | .. autosummary::
43 | :toctree: _generated/
44 | :template: function.rst
45 |
46 | get_default_layer_name
47 | validate_schema_names
48 | validate_table_name
49 |
--------------------------------------------------------------------------------
/docs/source/latexindex.rst:
--------------------------------------------------------------------------------
1 | :orphan:
2 |
3 | ######################
4 | PyDriosm Documentation
5 | ######################
6 |
7 | .. toctree::
8 | :maxdepth: 1
9 | :includehidden:
10 | :caption: Documentation
11 |
12 | introduction
13 | installation
14 | modules
15 | license
16 | acknowledgement
17 | contributors
18 |
19 | .. toctree::
20 | :maxdepth: 2
21 | :includehidden:
22 | :caption: Tutorial
23 |
24 | quick-start
25 |
--------------------------------------------------------------------------------
/docs/source/license.rst:
--------------------------------------------------------------------------------
1 | =======
2 | License
3 | =======
4 |
5 | - **PyDriosm**
6 |
7 | - `PyDriosm `_ is licensed under `GNU General Public License v3.0 `_ or later (GPLv3+).
8 |
9 | - **OpenStreetMap data**
10 |
11 | - The free `OpenStreetMap `_ data, which is used for the development of PyDriosm, is licensed under the `Open Data Commons Open Database License `_ (ODbL) by the `OpenStreetMap Foundation `_ (OSMF).
12 | - For more details about the use of the OpenStreetMap data, refer to the web page of `Copyright and Licence `_.
13 |
--------------------------------------------------------------------------------
/docs/source/modules.rst:
--------------------------------------------------------------------------------
1 | ======================
2 | Sub-packages / modules
3 | ======================
4 |
5 | .. py:module:: pydriosm
6 |
7 | The package includes the following sub-packages and modules:
8 |
9 | Sub-packages
10 | ============
11 |
12 | .. autosummary::
13 |
14 | downloader
15 | reader
16 | ios
17 |
18 | .. toctree::
19 | :maxdepth: 2
20 | :hidden:
21 |
22 | downloader
23 | reader
24 | ios
25 |
26 | Modules
27 | =======
28 |
29 | .. autosummary::
30 |
31 | errors
32 | utils
33 |
34 | .. toctree::
35 | :maxdepth: 2
36 | :hidden:
37 |
38 | errors
39 | utils
40 |
--------------------------------------------------------------------------------
/docs/source/reader.rst:
--------------------------------------------------------------------------------
1 | reader
2 | ======
3 |
4 | .. py:module:: pydriosm.reader
5 |
6 | .. automodule:: pydriosm.reader
7 | :noindex:
8 | :no-members:
9 | :no-undoc-members:
10 | :no-inherited-members:
11 |
12 |
13 | Transform OSM data
14 | ------------------
15 |
16 | .. autosummary::
17 | :toctree: _generated/
18 | :template: class.rst
19 |
20 | Transformer
21 |
22 | Parse OSM data
23 | --------------
24 |
25 | .. autosummary::
26 | :toctree: _generated/
27 | :template: class.rst
28 |
29 | SHPReadParse
30 | PBFReadParse
31 | VarReadParse
32 |
33 | Read OSM data
34 | -------------
35 |
36 | .. autosummary::
37 | :toctree: _generated/
38 | :template: class.rst
39 |
40 | GeofabrikReader
41 | BBBikeReader
42 |
--------------------------------------------------------------------------------
/docs/source/rtd-req.txt:
--------------------------------------------------------------------------------
1 | pyhelpers
2 | pyrcs
3 | pyshp
4 | Sphinx==7.2.6
5 | sphinx-copybutton==0.5.2
6 | sphinx-rtd-theme==1.3.0
7 |
--------------------------------------------------------------------------------
/docs/source/utils.rst:
--------------------------------------------------------------------------------
1 | utils
2 | =====
3 |
4 | .. py:module:: pydriosm.utils
5 |
6 | .. automodule:: pydriosm.utils
7 | :noindex:
8 | :no-members:
9 | :no-undoc-members:
10 | :no-inherited-members:
11 |
12 | Check data pathnames
13 | --------------------
14 |
15 | .. autosummary::
16 | :toctree: _generated/
17 | :template: function.rst
18 |
19 | check_relpath
20 | cdd_geofabrik
21 | cdd_bbbike
22 |
23 | General utilities
24 | -----------------
25 |
26 | .. autosummary::
27 | :toctree: _generated/
28 | :template: function.rst
29 |
30 | first_unique
31 | check_json_engine
32 | remove_osm_file
33 |
--------------------------------------------------------------------------------
/pydriosm/__init__.py:
--------------------------------------------------------------------------------
1 | """
2 | Package initialization.
3 | """
4 |
5 | import datetime
6 | import json
7 | import pkgutil
8 |
9 | from .downloader import BBBikeDownloader, GeofabrikDownloader
10 | from .ios import PostgresOSM
11 | from .reader import BBBikeReader, GeofabrikReader, PBFReadParse, SHPReadParse, VarReadParse
12 |
13 | metadata = json.loads(pkgutil.get_data(__name__, "data/metadata.json").decode())
14 |
15 | __project__ = metadata['Project']
16 | __pkgname__ = metadata['Package']
17 |
18 | __author__ = metadata['Author']
19 | __affiliation__ = metadata['Affiliation']
20 | __author_email__ = metadata['Email']
21 |
22 | __description__ = metadata['Description']
23 |
24 | __copyright__ = f'2019-{datetime.datetime.now().year}, {__author__}'
25 |
26 | __version__ = metadata['Version']
27 | __license__ = metadata['License']
28 |
29 | __first_release_date__ = metadata['First release']
30 |
31 | __all__ = [
32 | __author__,
33 | __description__,
34 | __author_email__,
35 | __pkgname__,
36 | __version__,
37 | 'downloader', 'GeofabrikDownloader', 'BBBikeDownloader',
38 | 'reader', 'PBFReadParse', 'SHPReadParse', 'VarReadParse', 'GeofabrikReader', 'BBBikeReader',
39 | 'ios', 'PostgresOSM',
40 | ]
41 |
--------------------------------------------------------------------------------
/pydriosm/_updater.py:
--------------------------------------------------------------------------------
1 | """
2 | Update (prepacked) package data.
3 | """
4 |
5 | import time
6 |
7 | from pyhelpers.ops import confirmed
8 |
9 | from .downloader import BBBikeDownloader, GeofabrikDownloader
10 |
11 |
12 | def _update_prepacked_data(verbose=True, interval=5):
13 | """
14 | Update prepacked data used by the downloader classes.
15 |
16 | :param verbose: whether to print relevant information in console, defaults to ``True``
17 | :type verbose: bool | int
18 | :param interval: time gap (in seconds) between the updating of different classes,
19 | defaults to ``5`` (seconds)
20 | :type interval: int | float
21 |
22 | **Examples**::
23 |
24 | >>> from pydriosm._updater import _update_prepacked_data
25 |
26 | >>> _update_prepacked_data(verbose=True)
27 | To update resources (which may take a few minutes)
28 | ? [No]|Yes: no
29 | """
30 |
31 | if confirmed("To update resources (which may take a few minutes)\n?"):
32 |
33 | meth_args = {
34 | 'update': True,
35 | 'confirmation_required': False,
36 | 'verbose': verbose,
37 | }
38 |
39 | # -- Geofabrik -----------------------------------------------------------------------------
40 | gfd = GeofabrikDownloader()
41 |
42 | _ = gfd.get_download_index(**meth_args)
43 |
44 | time.sleep(interval)
45 |
46 | _ = gfd.get_continent_tables(**meth_args)
47 |
48 | time.sleep(interval)
49 |
50 | _ = gfd.get_region_subregion_tier(**meth_args)
51 |
52 | time.sleep(interval)
53 |
54 | _ = gfd.get_catalogue(**meth_args)
55 |
56 | time.sleep(interval)
57 |
58 | _ = gfd.get_valid_subregion_names(**meth_args)
59 |
60 | time.sleep(interval)
61 |
62 | # -- BBBike --------------------------------------------------------------------------------
63 | bbd = BBBikeDownloader()
64 |
65 | _ = bbd.get_names_of_cities(**meth_args)
66 |
67 | time.sleep(interval)
68 |
69 | _ = bbd.get_coordinates_of_cities(**meth_args)
70 |
71 | time.sleep(interval)
72 |
73 | _ = bbd.get_subregion_index(**meth_args)
74 |
75 | time.sleep(interval)
76 |
77 | _ = bbd.get_valid_subregion_names(**meth_args)
78 |
79 | time.sleep(interval)
80 |
81 | _ = bbd.get_catalogue(**meth_args)
82 |
83 | if verbose:
84 | print("\nUpdate finished.")
85 |
--------------------------------------------------------------------------------
/pydriosm/data/bbbike_cities.pkl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/pydriosm/data/bbbike_cities.pkl
--------------------------------------------------------------------------------
/pydriosm/data/bbbike_cities_coordinates.pkl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/pydriosm/data/bbbike_cities_coordinates.pkl
--------------------------------------------------------------------------------
/pydriosm/data/bbbike_downloads_catalogue.pkl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/pydriosm/data/bbbike_downloads_catalogue.pkl
--------------------------------------------------------------------------------
/pydriosm/data/bbbike_index_of_subregions.pkl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/pydriosm/data/bbbike_index_of_subregions.pkl
--------------------------------------------------------------------------------
/pydriosm/data/bbbike_subregion_names.pkl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/pydriosm/data/bbbike_subregion_names.pkl
--------------------------------------------------------------------------------
/pydriosm/data/geofabrik_continent_tables.pkl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/pydriosm/data/geofabrik_continent_tables.pkl
--------------------------------------------------------------------------------
/pydriosm/data/geofabrik_downloads_catalogue.pkl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/pydriosm/data/geofabrik_downloads_catalogue.pkl
--------------------------------------------------------------------------------
/pydriosm/data/geofabrik_index_of_subregions.pkl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/pydriosm/data/geofabrik_index_of_subregions.pkl
--------------------------------------------------------------------------------
/pydriosm/data/geofabrik_region-subregion_tier.pkl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/pydriosm/data/geofabrik_region-subregion_tier.pkl
--------------------------------------------------------------------------------
/pydriosm/data/geofabrik_subregion_names.pkl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/pydriosm/data/geofabrik_subregion_names.pkl
--------------------------------------------------------------------------------
/pydriosm/data/metadata.json:
--------------------------------------------------------------------------------
1 | {
2 | "Project": "PyDriosm",
3 | "Package": "pydriosm",
4 | "Author": "Qian Fu",
5 | "Affiliation": "Birmingham Centre for Railway Research and Education, School of Engineering, University of Birmingham",
6 | "Email": "q.fu@bham.ac.uk",
7 | "Description": "An open-source tool for downloading, reading and PostgreSQL-based I/O of OpenStreetMap data.",
8 | "Version": "2.3.0b1",
9 | "License": "GPLv3+",
10 | "First release": "September 2019"
11 | }
--------------------------------------------------------------------------------
/pydriosm/downloader/__init__.py:
--------------------------------------------------------------------------------
1 | """
2 | Download `OpenStreetMap `_ (OSM) data from free download servers:
3 | `Geofabrik `_ and `BBBike `_.
4 | """
5 |
6 | from .bbbike import BBBikeDownloader
7 | from .geofabrik import GeofabrikDownloader
8 |
9 | __all__ = [
10 | 'GeofabrikDownloader',
11 | 'BBBikeDownloader',
12 | ]
13 |
--------------------------------------------------------------------------------
/pydriosm/errors.py:
--------------------------------------------------------------------------------
1 | """
2 | Define custom errors/exceptions.
3 | """
4 |
5 |
6 | class InvalidSubregionNameError(Exception):
7 | """
8 | Exception raised when an input `subregion_name` is not recognizable.
9 | """
10 |
11 | def __init__(self, subregion_name, msg=None):
12 | """
13 | :param subregion_name: name of a (sub)region available on a free download server
14 | :type subregion_name: str
15 | :param msg: index of optional messages, defaults to ``None``; options include {1, 2}
16 | :type msg: int | None
17 |
18 | :ivar: str subregion_name: name of a (sub)region available on a free download server
19 | :ivar: int | None msg: index of optional messages; options include {1, 2}
20 | :ivar: str: error message
21 |
22 | **Examples**::
23 |
24 | >>> from pydriosm.errors import InvalidSubregionNameError
25 |
26 | >>> raise InvalidSubregionNameError(subregion_name='abc')
27 | Traceback (most recent call last):
28 | ...
29 | pydriosm.errors.InvalidSubregionNameError:
30 | `subregion_name='abc'` -> The input of `subregion_name` is not recognizable.
31 | Check the `.data_source`, or try another one instead.
32 |
33 | >>> from pydriosm.downloader import GeofabrikDownloader, BBBikeDownloader
34 |
35 | >>> gfd = GeofabrikDownloader()
36 | >>> gfd.validate_subregion_name(subregion_name='birmingham')
37 | Traceback (most recent call last):
38 | ...
39 | pydriosm.errors.InvalidSubregionNameError:
40 | `subregion_name='birmingham'`
41 | 1) `subregion_name` fails to match any in `.valid_subregion_names`; or
42 | 2) The queried (sub)region is not available on the free download server.
43 |
44 | >>> bbd = BBBikeDownloader()
45 | >>> bbd.validate_subregion_name(subregion_name='bham')
46 | Traceback (most recent call last):
47 | ...
48 | pydriosm.errors.InvalidSubregionNameError:
49 | `subregion_name='bham'` -> The input of `subregion_name` is not recognizable.
50 | Check the `.data_source`, or try another one instead.
51 | """
52 |
53 | self.subregion_name = subregion_name
54 | self.msg = msg
55 |
56 | if self.msg == 1:
57 | self.message = \
58 | "\t1) `subregion_name` fails to match any in " \
59 | "`.valid_subregion_names`; " \
60 | "or\n" \
61 | "\t2) The queried (sub)region is not available on the free download server."
62 | else:
63 | self.message = "The input of `subregion_name` is not recognizable.\n" \
64 | " Check the `.data_source`, or try another one instead."
65 |
66 | super().__init__(self.message)
67 |
68 | def __str__(self):
69 | conj = "\n" if self.msg == 1 else " -> "
70 | return f"\n `subregion_name='{self.subregion_name}'`{conj}{self.message}"
71 |
72 |
73 | class InvalidFileFormatError(Exception):
74 | """
75 | Exception raised when an input `osm_file_format` is not recognizable.
76 | """
77 |
78 | def __init__(self, osm_file_format, valid_file_formats=None):
79 | """
80 | :param osm_file_format: file format/extension of the OSM data on the free download server
81 | :type osm_file_format: str
82 | :param valid_file_formats: filename extensions of the data files available on
83 | the free download server, defaults to ``None``
84 | :type valid_file_formats: typing.Iterable | None
85 |
86 | :ivar: str osm_file_format: file format/extension of the OSM data
87 | available on the free download server
88 | :ivar: int | None message: error message
89 |
90 | **Examples**::
91 |
92 | >>> from pydriosm.errors import InvalidFileFormatError
93 |
94 | >>> raise InvalidFileFormatError(osm_file_format='abc')
95 | Traceback (most recent call last):
96 | ...
97 | pydriosm.errors.InvalidFileFormatError:
98 | `osm_file_format='abc'` -> The input `osm_file_format` is unidentifiable.
99 |
100 | >>> from pydriosm.downloader import GeofabrikDownloader, BBBikeDownloader
101 |
102 | >>> gfd = GeofabrikDownloader()
103 | >>> gfd.validate_file_format(osm_file_format='abc')
104 | Traceback (most recent call last):
105 | ...
106 | pydriosm.errors.InvalidFileFormatError:
107 | `osm_file_format='abc'` -> The input `osm_file_format` is unidentifiable.
108 | Valid options include: {'.shp.zip', '.osm.pbf', '.osm.bz2'}.
109 |
110 | >>> bbd = BBBikeDownloader()
111 | >>> bbd.validate_file_format(osm_file_format='abc')
112 | Traceback (most recent call last):
113 | ...
114 | pydriosm.errors.InvalidFileFormatError:
115 | `osm_file_format='abc'` -> The input `osm_file_format` is unidentifiable.
116 | Valid options include: {'.shp.zip', '.geojson.xz', '.mapsforge-osm.zip', '.pbf', ...
117 | """
118 |
119 | self.osm_file_format = osm_file_format
120 |
121 | self.message = "The input `osm_file_format` is unidentifiable."
122 | if valid_file_formats:
123 | self.message += f"\n\tValid options include: {valid_file_formats}."
124 |
125 | super().__init__(self.message)
126 |
127 | def __str__(self):
128 | return f"\n `osm_file_format='{self.osm_file_format}'` -> {self.message}"
129 |
130 |
131 | class OtherTagsReformatError(Exception):
132 | """
133 | Exception raised when errors occur in the process of parsing ``other_tags`` in a PBF data file.
134 | """
135 |
136 | def __init__(self, other_tags):
137 | """
138 | :param other_tags: data of ``'other_tags'`` of a single feature in a PBF data file
139 | :type other_tags: str | None
140 |
141 | :ivar str | None other_tags: data of ``'other_tags'`` of a single feature
142 | in a PBF data file
143 | :ivar str message: error message
144 |
145 | **Examples**::
146 |
147 | >>> from pydriosm.errors import OtherTagsReformatError
148 |
149 | >>> raise OtherTagsReformatError(other_tags='abc')
150 | Traceback (most recent call last):
151 | ...
152 | pydriosm.errors.OtherTagsReformatError:
153 | `other_tags='abc'` -> Failed to reformat the ...
154 | """
155 |
156 | self.other_tags = other_tags
157 | self.message = "Failed to reformat the `other_tags`."
158 |
159 | super().__init__(self.message)
160 |
161 | def __str__(self):
162 | return f"\n `other_tags='{self.other_tags}'` -> {self.message}"
163 |
--------------------------------------------------------------------------------
/pydriosm/ios/__init__.py:
--------------------------------------------------------------------------------
1 | """
2 | Implement storage I/O of (parsed) OSM data extracts with `PostgreSQL `_.
3 | """
4 |
5 | from ._ios import PostgresOSM
6 | from .bbbike import BBBikeIOS
7 | from .geofabrik import GeofabrikIOS
8 |
9 | __all__ = ['PostgresOSM', 'GeofabrikIOS', 'BBBikeIOS']
10 |
--------------------------------------------------------------------------------
/pydriosm/ios/bbbike.py:
--------------------------------------------------------------------------------
1 | """
2 | Implement storage I/O of (parsed) OSM data extracts (available from BBBike free download server)
3 | with `PostgreSQL `_.
4 | """
5 |
6 | from pydriosm.ios._ios import PostgresOSM
7 |
8 |
9 | class BBBikeIOS:
10 | """
11 | Implement storage I/O of `BBBike exports of OpenStreetMap data `_
12 | with PostgreSQL.
13 | """
14 |
15 | def __init__(self, **kwargs):
16 | """
17 | :param kwargs: [optional] parameters of the class :class:`~pydriosm.downloader.PostgresOSM`
18 |
19 | :ivar BBBikeDownloader downloader: instance of the class
20 | :class:`~pydriosm.downloader.BBBikeDownloader`
21 | :ivar BBBikeReader reader: instance of the class
22 | :class:`~pydriosm.downloader.BBBikeReader`
23 |
24 | **Examples**::
25 |
26 | >>> from pydriosm.ios import BBBikeIOS
27 |
28 | >>> bbi = BBBikeIOS(database_name='osmdb_test')
29 | Password (postgres@localhost:5432): ***
30 | Creating a database: "osmdb_test" ... Done.
31 | Connecting postgres:***@localhost:5432/osmdb_test ... Successfully.
32 |
33 | >>> type(bbi.dbms)
34 | pydriosm.ios.PostgresOSM
35 |
36 | >>> bbi.dbms.name
37 | 'BBBike exports of OpenStreetMap data'
38 |
39 | .. seealso::
40 |
41 | - Examples for all the methods of the class :class:`~pydriosm.ios.PostgresOSM`.
42 | """
43 |
44 | kwargs.update({'data_source': 'BBBike'})
45 | self.dbms = PostgresOSM(**kwargs)
46 |
--------------------------------------------------------------------------------
/pydriosm/ios/geofabrik.py:
--------------------------------------------------------------------------------
1 | """
2 | Implement storage I/O of (parsed) OSM data extracts (available from Geofabrik free download server)
3 | with `PostgreSQL `_.
4 | """
5 |
6 | from pydriosm.ios._ios import PostgresOSM
7 |
8 |
9 | class GeofabrikIOS:
10 | """
11 | Implement storage I/O of `Geofabrik OpenStreetMap data extracts `_
12 | with `PostgreSQL `_.
13 | """
14 |
15 | def __init__(self, **kwargs):
16 | """
17 | :param kwargs: [optional] parameters of the class :class:`~pydriosm.downloader.PostgresOSM`
18 |
19 | :ivar PostgresOSM postgres: instance of the class :class:`~pydriosm.downloader.PostgresOSM`
20 | :ivar GeofabrikDownloader downloader: instance of the class
21 | :class:`~pydriosm.downloader.GeofabrikDownloader`
22 | :ivar GeofabrikReader reader: instance of the class
23 | :class:`~pydriosm.downloader.GeofabrikReader`
24 |
25 | **Examples**::
26 |
27 | >>> from pydriosm.ios import GeofabrikIOS
28 |
29 | >>> gfi = GeofabrikIOS(database_name='osmdb_test')
30 | Password (postgres@localhost:5432): ***
31 | Creating a database: "osmdb_test" ... Done.
32 | Connecting postgres:***@localhost:5432/osmdb_test ... Successfully.
33 |
34 | >>> type(gfi.dbms)
35 | pydriosm.ios.PostgresOSM
36 |
37 | >>> gfi.dbms.name
38 | 'Geofabrik OpenStreetMap data extracts'
39 |
40 | .. seealso::
41 |
42 | - Examples for all the methods of the class :class:`~pydriosm.ios.PostgresOSM`.
43 | """
44 |
45 | kwargs.update({'data_source': 'Geofabrik'})
46 | self.dbms = PostgresOSM(**kwargs)
47 |
--------------------------------------------------------------------------------
/pydriosm/ios/utils.py:
--------------------------------------------------------------------------------
1 | """
2 | Utilities for the :mod:`~pydriosm.ios` module.
3 | """
4 |
5 | from pyhelpers.text import find_similar_str, remove_punctuation
6 |
7 | from pydriosm.reader import PBFReadParse, SHPReadParse
8 |
9 |
10 | def get_default_layer_name(schema_name):
11 | """
12 | Get default name (as an input schema name) of an OSM layer
13 | for the class :py:class:`PostgresOSM`.
14 |
15 | See, for example, the method :meth:`pydriosm.ios.PostgresOSM.import_osm_layer`.
16 |
17 | :param schema_name: name of a schema (or name of an OSM layer)
18 | :type schema_name: str
19 | :return: default name of the layer
20 | :rtype: str
21 |
22 | **Examples**::
23 |
24 | >>> from pydriosm.ios import get_default_layer_name
25 |
26 | >>> lyr_name = get_default_layer_name(schema_name='point')
27 | >>> lyr_name
28 | 'points'
29 |
30 | >>> lyr_name = get_default_layer_name(schema_name='land')
31 | >>> lyr_name
32 | 'landuse'
33 | """
34 |
35 | valid_layer_names = set(PBFReadParse.LAYER_GEOM.keys()).union(SHPReadParse.LAYER_NAMES)
36 |
37 | layer_name_ = find_similar_str(x=schema_name, lookup_list=valid_layer_names)
38 |
39 | return layer_name_
40 |
41 |
42 | def validate_schema_names(schema_names=None, schema_named_as_layer=False):
43 | """
44 | Validate schema names for importing data into a database.
45 |
46 | :param schema_names: one or multiple names of layers, e.g. 'points', 'lines', defaults to ``None``
47 | :type schema_names: typing.Iterable | None
48 | :param schema_named_as_layer: whether to use default PBF layer name as the schema name,
49 | defaults to ``False``
50 | :type schema_named_as_layer: bool
51 | :return: valid names of the schemas in the database
52 | :rtype: list
53 |
54 | **Examples**::
55 |
56 | >>> from pydriosm.ios import validate_schema_names
57 |
58 | >>> valid_names = validate_schema_names()
59 | >>> valid_names
60 | []
61 |
62 | >>> input_schema_names = ['point', 'polygon']
63 | >>> valid_names = validate_schema_names(input_schema_names)
64 | >>> valid_names
65 | ['point', 'polygon']
66 |
67 | >>> valid_names = validate_schema_names(input_schema_names, schema_named_as_layer=True)
68 | >>> valid_names
69 | ['points', 'multipolygons']
70 | """
71 |
72 | if schema_names:
73 | if isinstance(schema_names, str):
74 | schema_names_ = [
75 | get_default_layer_name(schema_names) if schema_named_as_layer else schema_names]
76 | # assert schema_names_[0] in valid_layer_names, assertion_msg
77 | else: # isinstance(schema_names, list) is True
78 | if schema_named_as_layer:
79 | schema_names_ = [get_default_layer_name(x) for x in schema_names]
80 | else:
81 | schema_names_ = schema_names
82 | else:
83 | schema_names_ = []
84 |
85 | return schema_names_
86 |
87 |
88 | def validate_table_name(table_name, sub_space=''):
89 | """
90 | Validate a table name for importing OSM data into a database.
91 |
92 | :param table_name: name as input of a table in a PostgreSQL database
93 | :type table_name: str
94 | :param sub_space: substitute for space, defaults to ``''``
95 | :type sub_space: str
96 | :return: valid name of the table in the database
97 | :rtype: str
98 |
99 | **Examples**::
100 |
101 | >>> from pydriosm.ios import validate_table_name
102 |
103 | >>> subrgn_name = 'greater london'
104 | >>> valid_table_name = validate_table_name(subrgn_name)
105 | >>> valid_table_name
106 | 'greater london'
107 |
108 | >>> subrgn_name = 'Llanfairpwllgwyngyllgogerychwyrndrobwllllantysiliogogogoch, Wales'
109 | >>> valid_table_name = validate_table_name(subrgn_name, sub_space='_')
110 | >>> valid_table_name
111 | 'Llanfairpwllgwyngyllgogerychwyrndrobwllllantysiliogogogoch_W..'
112 | """
113 |
114 | table_name_ = remove_punctuation(x=table_name, rm_whitespace=True)
115 |
116 | if sub_space:
117 | table_name_ = table_name_.replace(' ', sub_space)
118 |
119 | table_name_ = table_name_[:60] + '..' if len(table_name_) >= 63 else table_name_
120 |
121 | return table_name_
122 |
--------------------------------------------------------------------------------
/pydriosm/reader/__init__.py:
--------------------------------------------------------------------------------
1 | """
2 | Read the OSM data extracts in various file formats.
3 | """
4 |
5 | from .bbbike import BBBikeReader
6 | from .geofabrik import GeofabrikReader
7 | from .parser import PBFReadParse, SHPReadParse, VarReadParse
8 | from .transformer import Transformer
9 |
10 | __all__ = [
11 | 'GeofabrikReader', 'BBBikeReader',
12 | 'Transformer',
13 | 'PBFReadParse', 'SHPReadParse', 'VarReadParse'
14 | ]
15 |
--------------------------------------------------------------------------------
/pydriosm/reader/bbbike.py:
--------------------------------------------------------------------------------
1 | """
2 | Read OpenStreetMap data extracts available from BBBike free download server.
3 | """
4 |
5 | from pydriosm.downloader import BBBikeDownloader
6 | from pydriosm.reader._reader import _Reader
7 |
8 |
9 | class BBBikeReader(_Reader):
10 | """
11 | Read `BBBike `_ exports of OpenStreetMap data.
12 | """
13 |
14 | #: str: Default download directory.
15 | DEFAULT_DOWNLOAD_DIR = "osm_data\\bbbike"
16 | #: set: Valid file formats.
17 | FILE_FORMATS = {
18 | '.csv.xz',
19 | '.garmin-onroad-latin1.zip',
20 | '.garmin-onroad.zip',
21 | '.garmin-opentopo.zip',
22 | '.garmin-osm.zip',
23 | '.geojson.xz',
24 | '.gz',
25 | '.mapsforge-osm.zip',
26 | '.pbf',
27 | '.shp.zip',
28 | '.svg-osm.zip',
29 | }
30 |
31 | def __init__(self, data_dir=None, max_tmpfile_size=None):
32 | """
33 | :param data_dir: (a path or a name of) a directory where a data file is;
34 | if ``None`` (default), a folder ``osm_bbbike`` under the current working directory
35 | :type data_dir: str | None
36 | :param max_tmpfile_size: defaults to ``None``,
37 | see also :func:`gdal_configurations`
38 | :type max_tmpfile_size: int | None
39 |
40 | :ivar BBBikeDownloader downloader: instance of the class
41 | :py:class:`BBBikeDownloader`
42 | :ivar str name: name of the data resource
43 | :ivar str url: url of the homepage to the BBBike free download server
44 |
45 | **Examples**::
46 |
47 | >>> from pydriosm.reader import BBBikeReader
48 |
49 | >>> bbr = BBBikeReader()
50 |
51 | >>> bbr.NAME
52 | 'BBBike'
53 | """
54 |
55 | # noinspection PyTypeChecker
56 | super().__init__(
57 | downloader=BBBikeDownloader, data_dir=data_dir, max_tmpfile_size=max_tmpfile_size)
58 |
59 | def read_osm_pbf(self, subregion_name, data_dir=None, readable=False, expand=False,
60 | parse_geometry=False, parse_other_tags=False, parse_properties=False,
61 | update=False, download=True, pickle_it=False, ret_pickle_path=False,
62 | rm_pbf_file=False, chunk_size_limit=50, verbose=False, **kwargs):
63 | """
64 | Read a PBF (.osm.pbf) data file of a geographic (sub)region.
65 |
66 | :param subregion_name: name of a geographic (sub)region (case-insensitive)
67 | that is available on Geofabrik free download server
68 | :type subregion_name: str
69 | :param data_dir: directory where the .osm.pbf data file is located/saved;
70 | if ``None``, the default local directory
71 | :type data_dir: str | None
72 | :param readable: whether to parse each feature in the raw data, defaults to ``False``
73 | :type readable: bool
74 | :param expand: whether to expand dict-like data into separate columns, defaults to ``False``
75 | :type expand: bool
76 | :param parse_geometry: whether to represent the ``'geometry'`` field
77 | in a `shapely.geometry`_ format, defaults to ``False``
78 | :type parse_geometry: bool
79 | :param parse_properties: whether to represent the ``'properties'`` field
80 | in a tabular format, defaults to ``False``
81 | :type parse_properties: bool
82 | :param parse_other_tags: whether to represent a ``'other_tags'`` (of ``'properties'``)
83 | in a `dict`_ format, defaults to ``False``
84 | :type parse_other_tags: bool
85 | :param download: whether to download/update the PBF data file of the given subregion,
86 | if it is not available at the specified path, defaults to ``True``
87 | :type download: bool
88 | :param update: whether to check to update pickle backup (if available), defaults to ``False``
89 | :type update: bool
90 | :param pickle_it: whether to save the .pbf data as a pickle file, defaults to ``False``
91 | :type pickle_it: bool
92 | :param ret_pickle_path: (when ``pickle_it=True``)
93 | whether to return a path to the saved pickle file
94 | :type ret_pickle_path: bool
95 | :param rm_pbf_file: whether to delete the downloaded .osm.pbf file, defaults to ``False``
96 | :type rm_pbf_file: bool
97 | :param chunk_size_limit: threshold (in MB) that triggers the use of chunk parser,
98 | defaults to ``50``;
99 | if the size of the .osm.pbf file (in MB) is greater than ``chunk_size_limit``,
100 | it will be parsed in a chunk-wise way
101 | :type chunk_size_limit: int | None
102 | :param verbose: whether to print relevant information in console as the function runs,
103 | defaults to ``False``
104 | :type verbose: bool | int
105 | :param kwargs: [optional] parameters of the method
106 | :meth:`_Reader.read_osm_pbf()`
107 | :return: dictionary of the .osm.pbf data;
108 | when ``pickle_it=True``, return a tuple of the dictionary and a path to the pickle file
109 | :rtype: dict | tuple | None
110 |
111 | .. _`shapely.geometry`:
112 | https://shapely.readthedocs.io/en/latest/manual.html#geometric-objects
113 | .. _`dict`:
114 | https://docs.python.org/3/library/stdtypes.html#dict
115 |
116 | **Examples**::
117 |
118 | >>> from pydriosm.reader import BBBikeReader
119 | >>> from pyhelpers.dirs import delete_dir
120 |
121 | >>> bbr = BBBikeReader()
122 |
123 | >>> subrgn_name = 'Leeds'
124 | >>> dat_dir = "tests\\osm_data"
125 |
126 | >>> leeds_pbf_raw = bbr.read_osm_pbf(subrgn_name, data_dir=dat_dir, verbose=True)
127 | Downloading "Leeds.osm.pbf"
128 | to "tests\\osm_data\\leeds\\" ... Done.
129 | Reading "tests\\osm_data\\leeds\\Leeds.osm.pbf" ... Done.
130 | >>> type(leeds_pbf_raw)
131 | dict
132 | >>> list(leeds_pbf_raw.keys())
133 | ['points', 'lines', 'multilinestrings', 'multipolygons', 'other_relations']
134 |
135 | >>> pbf_raw_points = leeds_pbf_raw['points']
136 | >>> type(pbf_raw_points)
137 | list
138 | >>> type(pbf_raw_points[0])
139 | osgeo.ogr.Feature
140 |
141 | >>> # (Parsing the data in this example might take up to a few minutes.)
142 | >>> leeds_pbf_parsed = bbr.read_osm_pbf(
143 | ... subrgn_name, data_dir=dat_dir, readable=True, expand=True,
144 | ... parse_geometry=True, parse_other_tags=True, parse_properties=True,
145 | ... verbose=True)
146 | Parsing "tests\\osm_data\\leeds\\Leeds.osm.pbf" ... Done.
147 |
148 | >>> list(leeds_pbf_parsed.keys())
149 | ['points', 'lines', 'multilinestrings', 'multipolygons', 'other_relations']
150 |
151 | >>> # Data of the 'multipolygons' layer
152 | >>> leeds_pbf_parsed_multipolygons = leeds_pbf_parsed['multipolygons']
153 | >>> leeds_pbf_parsed_multipolygons.head()
154 | id geometry ... tourism other_tags
155 | 0 10595 (POLYGON ((-1.5030223 53.6725382, -1.5034495 5... ... None None
156 | 1 10600 (POLYGON ((-1.5116994 53.6764287, -1.5099361 5... ... None None
157 | 2 10601 (POLYGON ((-1.5142403 53.6710831, -1.5143686 5... ... None None
158 | 3 10612 (POLYGON ((-1.5129341 53.6704885, -1.5131883 5... ... None None
159 | 4 10776 (POLYGON ((-1.5523801 53.7029081, -1.5524772 5... ... None None
160 | [5 rows x 26 columns]
161 |
162 | >>> # Delete the example data and the test data directory
163 | >>> delete_dir(dat_dir, verbose=True)
164 | To delete the directory "tests\\osm_data\\" (Not empty)
165 | ? [No]|Yes: yes
166 | Deleting "tests\\osm_data\\" ... Done.
167 |
168 | .. seealso::
169 |
170 | - Examples for the method
171 | :meth:`GeofabrikReader.read_osm_pbf()`.
172 | """
173 |
174 | osm_pbf_data = super().read_osm_pbf(
175 | subregion_name=subregion_name, data_dir=data_dir, readable=readable, expand=expand,
176 | parse_geometry=parse_geometry, parse_properties=parse_properties,
177 | parse_other_tags=parse_other_tags, update=update, download=download,
178 | pickle_it=pickle_it, ret_pickle_path=ret_pickle_path, rm_pbf_file=rm_pbf_file,
179 | chunk_size_limit=chunk_size_limit, verbose=verbose, **kwargs)
180 |
181 | return osm_pbf_data
182 |
183 | def read_shp_zip(self, subregion_name, layer_names=None, feature_names=None, data_dir=None,
184 | update=False, download=True, pickle_it=False, ret_pickle_path=False,
185 | rm_extracts=False, rm_shp_zip=False, verbose=False, **kwargs):
186 | """
187 | Read a shapefile of a geographic (sub)region.
188 |
189 | :param subregion_name: name of a geographic (sub)region (case-insensitive)
190 | that is available on BBBike free download server
191 | :type subregion_name: str
192 | :param layer_names: name of a .shp layer, e.g. 'railways', or names of multiple layers;
193 | if ``None`` (default), all available layers
194 | :type layer_names: str | list | None
195 | :param feature_names: name of a feature, e.g. 'rail', or names of multiple features;
196 | if ``None`` (default), all available features
197 | :type feature_names: str | list | None
198 | :param data_dir: directory where the .shp.zip data file is located/saved;
199 | if ``None``, the default directory
200 | :type data_dir: str | None
201 | :param update: whether to check to update pickle backup (if available), defaults to ``False``
202 | :type update: bool
203 | :param download: whether to ask for confirmation
204 | before starting to download a file, defaults to ``True``
205 | :type download: bool
206 | :param pickle_it: whether to save the .shp data as a pickle file, defaults to ``False``
207 | :type pickle_it: bool
208 | :param ret_pickle_path: (when ``pickle_it=True``)
209 | whether to return a path to the saved pickle file
210 | :type ret_pickle_path: bool
211 | :param rm_extracts: whether to delete extracted files from the .shp.zip file,
212 | defaults to ``False``
213 | :type rm_extracts: bool
214 | :param rm_shp_zip: whether to delete the downloaded .shp.zip file, defaults to ``False``
215 | :type rm_shp_zip: bool
216 | :param verbose: whether to print relevant information in console as the function runs,
217 | defaults to ``False``
218 | :type verbose: bool | int
219 | :return: dictionary of the shapefile data, with keys and values being layer names
220 | and tabular data (in the format of `geopandas.GeoDataFrame`_), respectively;
221 | when ``pickle_it=True``, return a tuple of the dictionary and a path to the pickle file
222 | :rtype: dict | collections.OrderedDict | tuple | None
223 |
224 | .. _`geopandas.GeoDataFrame`: https://geopandas.org/reference.html#geodataframe
225 |
226 | **Examples**::
227 |
228 | >>> from pydriosm.reader import BBBikeReader
229 | >>> from pyhelpers.dirs import delete_dir
230 | >>> import os
231 |
232 | >>> bbr = BBBikeReader()
233 |
234 | >>> subrgn_name = 'Birmingham'
235 | >>> dat_dir = "tests\\osm_data"
236 |
237 | >>> bham_shp = bbr.read_shp_zip(
238 | ... subregion_name=subrgn_name, data_dir=dat_dir, download=False, verbose=True)
239 | The .shp.zip file for "Birmingham" is not found.
240 |
241 | >>> # Set `download=True`
242 | >>> bham_shp = bbr.read_shp_zip(
243 | ... subregion_name=subrgn_name, data_dir=dat_dir, download=True, verbose=True)
244 | Downloading "Birmingham.osm.shp.zip"
245 | to "tests\\osm_data\\birmingham\\" ... Done.
246 | Extracting "tests\\osm_data\\birmingham\\Birmingham.osm.shp.zip"
247 | to "tests\\osm_data\\birmingham\\" ... Done.
248 | Reading the shapefile(s) at
249 | "tests\\osm_data\\birmingham\\Birmingham-shp\\shape\\" ... Done.
250 | >>> type(bham_shp)
251 | collections.OrderedDict
252 | >>> list(bham_shp.keys())
253 | ['buildings',
254 | 'landuse',
255 | 'natural',
256 | 'places',
257 | 'points',
258 | 'railways',
259 | 'roads',
260 | 'waterways']
261 |
262 | >>> # Data of 'railways' layer
263 | >>> bham_railways_shp = bham_shp['railways']
264 | >>> bham_railways_shp.head()
265 | osm_id ... shape_type
266 | 0 740 ... 3
267 | 1 2148 ... 3
268 | 2 2950000 ... 3
269 | 3 3491845 ... 3
270 | 4 3981454 ... 3
271 | [5 rows x 5 columns]
272 |
273 | >>> # Read data of 'road' layer only from the original .shp.zip file
274 | >>> # (and delete all extracts)
275 | >>> lyr_name = 'roads'
276 | >>> bham_roads_shp = bbr.read_shp_zip(
277 | ... subregion_name=subrgn_name, layer_names=lyr_name, data_dir=dat_dir,
278 | ... rm_extracts=True, verbose=True)
279 | Reading "tests\\osm_data\\birmingham\\Birmingham-shp\\shape\\roads.shp" ... Done.
280 | Deleting the extracts "tests\\osm_data\\birmingham\\Birmingham-shp\\" ... Done.
281 | >>> type(bham_roads_shp)
282 | collections.OrderedDict
283 | >>> list(bham_roads_shp.keys())
284 | ['roads']
285 | >>> bham_roads_shp[lyr_name].head()
286 | osm_id ... shape_type
287 | 0 37 ... 3
288 | 1 38 ... 3
289 | 2 41 ... 3
290 | 3 45 ... 3
291 | 4 46 ... 3
292 | [5 rows x 9 columns]
293 |
294 | >>> # Read data of multiple layers and features from the original .shp.zip file
295 | >>> # (and delete all extracts)
296 | >>> lyr_names = ['railways', 'waterways']
297 | >>> feat_names = ['rail', 'canal']
298 | >>> bham_rw_rc_shp = bbr.read_shp_zip(
299 | ... subregion_name=subrgn_name, layer_names=lyr_names, feature_names=feat_names,
300 | ... data_dir=dat_dir, rm_extracts=True, rm_shp_zip=True, verbose=True)
301 | Extracting the following layer(s):
302 | 'railways'
303 | 'waterways'
304 | from "tests\\osm_data\\birmingham\\Birmingham.osm.shp.zip"
305 | to "tests\\osm_data\\birmingham\\" ... Done.
306 | Reading the data at "tests\\osm_data\\birmingham\\Birmingham-shp\\shape\\" ... Done.
307 | Deleting the extracts "tests\\osm_data\\birmingham\\Birmingham-shp\\" ... Done.
308 | Deleting "tests\\osm_data\\birmingham\\Birmingham.osm.shp.zip" ... Done.
309 | >>> type(bham_rw_rc_shp)
310 | collections.OrderedDict
311 | >>> list(bham_rw_rc_shp.keys())
312 | ['railways', 'waterways']
313 |
314 | >>> # Data of the 'railways' layer
315 | >>> bham_rw_rc_shp_railways = bham_rw_rc_shp['railways']
316 | >>> bham_rw_rc_shp_railways[['type', 'name']].head()
317 | type name
318 | 0 rail Cross-City Line
319 | 1 rail Cross-City Line
320 | 2 rail Derby to Birmingham (Proof House Junction) Line
321 | 3 rail Birmingham to Peterborough Line
322 | 4 rail Water Orton to Park Lane Junction Curve
323 |
324 | >>> # Data of the 'waterways' layer
325 | >>> bham_rw_rc_shp_waterways = bham_rw_rc_shp['waterways']
326 | >>> bham_rw_rc_shp_waterways[['type', 'name']].head()
327 | type name
328 | 2 canal Birmingham and Fazeley Canal
329 | 8 canal Birmingham and Fazeley Canal
330 | 9 canal Birmingham Old Line Canal Navigations - Rotton P
331 | 10 canal Oozells Street Loop
332 | 11 canal Worcester & Birmingham Canal
333 |
334 | >>> # Delete the example data and the test data directory
335 | >>> delete_dir(dat_dir, verbose=True)
336 | To delete the directory "tests\\osm_data\\" (Not empty)
337 | ? [No]|Yes: yes
338 | Deleting "tests\\osm_data\\" ... Done.
339 | """
340 |
341 | shp_data = super().read_shp_zip(
342 | subregion_name=subregion_name, layer_names=layer_names, feature_names=feature_names,
343 | data_dir=data_dir, update=update, download=download, pickle_it=pickle_it,
344 | ret_pickle_path=ret_pickle_path, rm_extracts=rm_extracts, rm_shp_zip=rm_shp_zip,
345 | verbose=verbose, **kwargs)
346 |
347 | return shp_data
348 |
349 | def read_csv_xz(self, subregion_name, data_dir=None, download=False, verbose=False, **kwargs):
350 | """
351 | Read a compressed CSV (.csv.xz) data file of a geographic (sub)region.
352 |
353 | :param subregion_name: name of a geographic (sub)region (case-insensitive)
354 | that is available on BBBike free download server
355 | :type subregion_name: str
356 | :param data_dir: directory where the .csv.xz data file is located/saved;
357 | if ``None`` (default), the default directory
358 | :type data_dir: str | None
359 | :param download: whether to try to download the requisite data file if it does not exist,
360 | defaults to ``True``
361 | :type download: bool
362 | :param verbose: whether to print relevant information in console as the function runs,
363 | defaults to ``False``
364 | :type verbose: bool | int
365 | :return: tabular data of the .csv.xz file
366 | :rtype: pandas.DataFrame | None
367 |
368 | .. _pydriosm-BBBikeReader-read_csv_xz:
369 |
370 | **Examples**::
371 |
372 | >>> from pydriosm.reader import BBBikeReader
373 | >>> from pyhelpers.dirs import cd, delete_dir
374 |
375 | >>> bbr = BBBikeReader()
376 |
377 | >>> subrgn_name = 'Leeds'
378 | >>> dat_dir = "tests\\osm_data"
379 |
380 | >>> leeds_csv_xz = bbr.read_csv_xz(subrgn_name, dat_dir, verbose=True)
381 | The requisite data file "tests\\osm_data\\leeds\\Leeds.osm.csv.xz" does not exist.
382 |
383 | >>> leeds_csv_xz = bbr.read_csv_xz(subrgn_name, dat_dir, verbose=True, download=True)
384 | Downloading "Leeds.osm.csv.xz"
385 | to "tests\\osm_data\\leeds\\" ... Done.
386 | Parsing the data ... Done.
387 |
388 | >>> leeds_csv_xz.head()
389 | type id feature note
390 | 0 node 154915 None None
391 | 1 node 154916 None None
392 | 2 node 154921 None None
393 | 3 node 154922 None None
394 | 4 node 154923 None None
395 |
396 | >>> # Delete the downloaded .csv.xz data file
397 | >>> delete_dir(dat_dir, verbose=True)
398 | To delete the directory "tests\\osm_data\\" (Not empty)
399 | ? [No]|Yes: yes
400 | Deleting "tests\\osm_data\\" ... Done.
401 | """
402 |
403 | csv_xz_data = self.read_osm_var(
404 | self.VAR.read_csv_xz, subregion_name=subregion_name, osm_file_format=".csv.xz",
405 | data_dir=data_dir, download=download, verbose=verbose, **kwargs)
406 |
407 | return csv_xz_data
408 |
409 | def read_geojson_xz(self, subregion_name, data_dir=None, parse_geometry=False, download=False,
410 | verbose=False, **kwargs):
411 | """
412 | Read a .geojson.xz data file of a geographic (sub)region.
413 |
414 | :param subregion_name: name of a geographic (sub)region (case-insensitive)
415 | that is available on BBBike free download server
416 | :type subregion_name: str
417 | :param data_dir: directory where the .geojson.xz data file is located/saved;
418 | if ``None`` (default), the default directory
419 | :type data_dir: str | None
420 | :param parse_geometry: whether to represent coordinates in a format of a geometric object,
421 | defaults to ``False``
422 | :type parse_geometry: bool
423 | :param download: whether to try to download the requisite data file if it does not exist,
424 | defaults to ``True``
425 | :type download: bool
426 | :param verbose: whether to print relevant information in console as the function runs,
427 | defaults to ``False``
428 | :type verbose: bool | int
429 | :return: tabular data of the .csv.xz file
430 | :rtype: pandas.DataFrame | None
431 |
432 | .. _pydriosm-BBBikeReader-read_geojson_xz:
433 |
434 | **Examples**::
435 |
436 | >>> from pydriosm.reader import BBBikeReader
437 | >>> from pyhelpers.dirs import cd, delete_dir
438 | >>> import os
439 |
440 | >>> bbr = BBBikeReader()
441 |
442 | >>> subrgn_name = 'Leeds'
443 | >>> dat_dir = "tests\\osm_data"
444 |
445 | >>> leeds_geoj = bbr.read_geojson_xz(subrgn_name, dat_dir, verbose=True)
446 | The requisite data file "tests\\osm_data\\leeds\\Leeds.osm.geojson.xz" does not exist.
447 |
448 | >>> # Set `try_download=True`
449 | >>> leeds_geoj = bbr.read_geojson_xz(subrgn_name, dat_dir, verbose=True, download=True)
450 | Downloading "Leeds.osm.geojson.xz"
451 | to "tests\\osm_data\\leeds\\" ... Done.
452 | Parsing the data ... Done.
453 | >>> leeds_geoj.head()
454 | geometry properties
455 | 0 {'type': 'Point', 'coordinates': [-1.5558097, ... {'highway': 'motorway_junction'...
456 | 1 {'type': 'Point', 'coordinates': [-1.34293, 53... {'highway': 'motorway_junction'...
457 | 2 {'type': 'Point', 'coordinates': [-1.517335, 5... {'highway': 'motorway_junction'...
458 | 3 {'type': 'Point', 'coordinates': [-1.514124, 5... {'highway': 'motorway_junction'...
459 | 4 {'type': 'Point', 'coordinates': [-1.516511, 5... {'highway': 'motorway_junction'...
460 |
461 | >>> # Set `parse_geometry` to be True
462 | >>> leeds_geoj_ = bbr.read_geojson_xz(subrgn_name, dat_dir, parse_geometry=True,
463 | ... verbose=True)
464 | Parsing "tests\\osm_data\\leeds\\Leeds.osm.geojson.xz" ... Done.
465 | >>> leeds_geoj_['geometry'].head()
466 | 0 POINT (-1.5560511 53.6879848)
467 | 1 POINT (-1.34293 53.844618)
468 | 2 POINT (-1.517335 53.7499667)
469 | 3 POINT (-1.514124 53.7416937)
470 | 4 POINT (-1.516511 53.7256632)
471 | Name: geometry, dtype: object
472 |
473 | >>> # Delete the download directory
474 | >>> delete_dir(dat_dir, verbose=True)
475 | """
476 |
477 | kwargs.update({'parse_geometry': parse_geometry})
478 |
479 | geojson_xz_data = self.read_osm_var(
480 | self.VAR.read_geojson_xz, subregion_name=subregion_name, osm_file_format=".geojson.xz",
481 | data_dir=data_dir, download=download, verbose=verbose, **kwargs)
482 |
483 | return geojson_xz_data
484 |
--------------------------------------------------------------------------------
/pydriosm/reader/transformer.py:
--------------------------------------------------------------------------------
1 | """
2 | Transform the OSM data.
3 | """
4 |
5 | import copy
6 | import re
7 |
8 | import pandas as pd
9 | import shapely.errors
10 | import shapely.geometry
11 | from pyhelpers.ops import update_dict
12 |
13 | from pydriosm.errors import OtherTagsReformatError
14 |
15 |
16 | class Transformer:
17 | """
18 | Transform / reformat data.
19 |
20 | **Examples**::
21 |
22 | >>> from pydriosm.reader import Transformer
23 |
24 | >>> geometry = {'type': 'Point', 'coordinates': [-0.5134241, 52.6555853]}
25 | >>> geometry_ = Transformer.transform_unitary_geometry(geometry)
26 | >>> type(geometry_)
27 | shapely.geometry.point.Point
28 | >>> geometry_.wkt
29 | 'POINT (-0.5134241 52.6555853)'
30 | """
31 |
32 | @classmethod
33 | def point_as_polygon(cls, multi_poly_coords):
34 | """
35 | Make the coordinates of a single 'Point' (in a 'MultiPolygon') be reformatted to
36 | a 'Polygon'-like coordinates.
37 |
38 | The list of coordinates of some 'MultiPolygon' features may contain single points.
39 | In order to reformat such multipart geometry (from dict into `shapely.geometry`_ type),
40 | there is a need to ensure each of the constituent parts is a `shapely.geometry.Polygon`_.
41 |
42 | :param multi_poly_coords: original data of coordinates of a
43 | `shapely.geometry.MultiPolygon`_ feature
44 | :type multi_poly_coords: list
45 | :return: coordinates that are reformatted as appropriate
46 | :rtype: list
47 |
48 | .. _`shapely.geometry`:
49 | https://shapely.readthedocs.io/en/latest/manual.html#geometric-objects
50 | .. _`shapely.geometry.Polygon`:
51 | https://shapely.readthedocs.io/en/stable/manual.html#Polygon
52 | .. _`shapely.geometry.MultiPolygon`:
53 | https://shapely.readthedocs.io/en/stable/manual.html#MultiPolygon
54 |
55 | **Examples**::
56 |
57 | >>> from pydriosm.reader import Transformer
58 |
59 | >>> geometry = {
60 | ... 'type': 'MultiPolygon',
61 | ... 'coordinates': [[[[-0.6920145, 52.6753268], [-0.6920145, 52.6753268]]]]
62 | ... }
63 | >>> mp_coords = geometry['coordinates']
64 |
65 | >>> mp_coords_ = Transformer.point_as_polygon(mp_coords)
66 | >>> mp_coords_
67 | [[[[-0.6920145, 52.6753268],
68 | [-0.6920145, 52.6753268],
69 | [-0.6920145, 52.6753268]]]]
70 | """
71 |
72 | coords = multi_poly_coords.copy()
73 | temp = coords[0][0]
74 |
75 | if len(temp) == 2 and temp[0] == temp[1]:
76 | coords[0][0] += [temp[0]]
77 |
78 | return coords
79 |
80 | @classmethod
81 | def transform_unitary_geometry(cls, geometry, mode=1, to_wkt=False):
82 | """
83 | Transform a unitary geometry from dict into a `shapely.geometry`_ object.
84 |
85 | :param geometry: geometry data for a feature of one of the geometry types including
86 | ``'Point'``, ``'LineString'``, ``'MultiLineString'`` and ``'MultiPolygon'``
87 | :type geometry: dict | pandas.DataFrame
88 | :param mode: indicate the way of parsing the input;
89 |
90 | - when ``mode=1`` **(default)**, the input ``geometry`` should be directly accessible
91 | and would be in the format of ``{'type': , 'coordinates': }``
92 | or as a row of a `pandas.DataFrame`_;
93 | - when ``mode=2``, the input ``geometry`` is in the `GeoJSON`_ format
94 |
95 | :type mode: int
96 | :param to_wkt: whether to represent the geometry in the WKT (well-known text) format,
97 | defaults to ``False``
98 | :type to_wkt: bool
99 | :return: reformatted geometry data
100 | :rtype: shapely.geometry.Point | dict | str
101 |
102 | .. _`shapely.geometry`:
103 | https://shapely.readthedocs.io/en/latest/manual.html#geometric-objects
104 | .. _`pandas.DataFrame`:
105 | https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.html
106 | .. _`GeoJSON`:
107 | https://geojson.org/
108 |
109 | **Examples**::
110 |
111 | >>> from pydriosm.reader import PBFReadParse
112 |
113 | >>> g1_dat = {'type': 'Point', 'coordinates': [-0.5134241, 52.6555853]}
114 | >>> g1_data = PBFReadParse.transform_unitary_geometry(g1_dat)
115 | >>> type(g1_data)
116 | shapely.geometry.point.Point
117 | >>> g1_data.wkt
118 | 'POINT (-0.5134241 52.6555853)'
119 |
120 | >>> g2_dat = {
121 | ... 'type': 'Feature',
122 | ... 'geometry': {
123 | ... 'type': 'Point',
124 | ... 'coordinates': [-0.5134241, 52.6555853]
125 | ... },
126 | ... 'properties': {
127 | ... 'osm_id': '488432',
128 | ... 'name': None,
129 | ... 'barrier': None,
130 | ... 'highway': None,
131 | ... 'ref': None,
132 | ... 'address': None,
133 | ... 'is_in': None,
134 | ... 'place': None,
135 | ... 'man_made': None,
136 | ... 'other_tags': '"odbl"=>"clean"'
137 | ... },
138 | ... 'id': 488432
139 | ... }
140 | >>> g2_data = PBFReadParse.transform_unitary_geometry(g2_dat, mode=2)
141 | >>> type(g2_data)
142 | dict
143 | >>> list(g2_data.keys())
144 | ['type', 'geometry', 'properties', 'id']
145 | >>> g2_data['geometry']
146 | 'POINT (-0.5134241 52.6555853)'
147 | """
148 |
149 | if mode == 1:
150 | geom_type, coords = geometry['type'], geometry['coordinates']
151 | geom_func = getattr(shapely.geometry, geom_type)
152 |
153 | if geom_type == 'MultiPolygon':
154 | geom_data = geom_func(
155 | shapely.geometry.Polygon(y) for x in cls.point_as_polygon(coords) for y in x)
156 | # geom_data = geom.wkt if to_wkt else geom.geoms
157 |
158 | else:
159 | geom_data = geom_func(coords)
160 | # if to_wkt:
161 | # geom_data = geom_data.wkt
162 | # elif 'Multi' in geom_type:
163 | # geom_data = geom_data.geoms
164 |
165 | if to_wkt:
166 | geom_data = geom_data.wkt
167 |
168 | else:
169 | geom_data = geometry.copy()
170 | dat = cls.transform_unitary_geometry(geometry['geometry'], mode=1, to_wkt=True)
171 | geom_data.update({'geometry': dat})
172 |
173 | return geom_data
174 |
175 | @classmethod
176 | def transform_geometry_collection(cls, geometry, mode=1, to_wkt=False):
177 | """
178 | Transform a collection of geometry from dict into a `shapely.geometry`_ object.
179 |
180 | :param geometry: geometry data for a feature of ``GeometryCollection``
181 | :type geometry: list | dict
182 | :param mode: indicate the way of parsing the input;
183 |
184 | - when ``mode=1`` **(default)**, the input ``geometry`` should be directly accessible
185 | and would be in the format of ``{'type': , 'coordinates': }``
186 | or as a row of a `pandas.DataFrame`_;
187 | - when ``mode=2``, the input ``geometry`` is in the `GeoJSON`_ format
188 |
189 | :type mode: int
190 | :param to_wkt: whether to represent the geometry in the WKT (well-known text) format,
191 | defaults to ``False``
192 | :type to_wkt: bool
193 | :return: reformatted geometry data
194 | :rtype: shapely.geometry.base.HeterogeneousGeometrySequence | dict | str
195 |
196 | .. _`shapely.geometry`:
197 | https://shapely.readthedocs.io/en/latest/manual.html#geometric-objects
198 | .. _`pandas.DataFrame`:
199 | https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.html
200 | .. _`GeoJSON`:
201 | https://geojson.org/
202 |
203 | **Examples**::
204 |
205 | >>> from pydriosm.reader import PBFReadParse
206 | >>> from shapely.geometry import GeometryCollection
207 |
208 | >>> g1_dat_ = {
209 | ... 'type': 'GeometryCollection',
210 | ... 'geometries': [
211 | ... {'type': 'Point', 'coordinates': [-0.5096176, 52.6605168]},
212 | ... {'type': 'Point', 'coordinates': [-0.5097337, 52.6605812]}
213 | ... ]
214 | ... }
215 | >>> g1_dat = g1_dat_['geometries']
216 | >>> g1_data = PBFReadParse.transform_geometry_collection(g1_dat)
217 | >>> type(g1_data)
218 | shapely.geometry.collection.GeometryCollection
219 | >>> g1_data.wkt
220 | 'GEOMETRYCOLLECTION (POINT (-0.5096176 52.6605168), POINT (-0.5097337 52.6605812))'
221 |
222 | >>> g2_dat = {
223 | ... 'type': 'Feature',
224 | ... 'geometry': {
225 | ... 'type': 'GeometryCollection',
226 | ... 'geometries': [
227 | ... {'type': 'Point', 'coordinates': [-0.5096176, 52.6605168]},
228 | ... {'type': 'Point', 'coordinates': [-0.5097337, 52.6605812]}]
229 | ... },
230 | ... 'properties': {
231 | ... 'osm_id': '256254',
232 | ... 'name': 'Fife Close',
233 | ... 'type': 'site',
234 | ... 'other_tags': '"naptan:StopAreaCode"=>"270G02701525"'
235 | ... },
236 | ... 'id': 256254
237 | ... }
238 | >>> g2_data = PBFReadParse.transform_geometry_collection(g2_dat, mode=2)
239 | >>> type(g2_data)
240 | dict
241 | >>> list(g2_data.keys())
242 | ['type', 'geometry', 'properties', 'id']
243 | >>> g2_data['geometry']
244 | 'GEOMETRYCOLLECTION (POINT (-0.5096176 52.6605168), POINT (-0.5097337 52.6605812))'
245 | """
246 |
247 | if mode == 1:
248 | geometry_collection = []
249 |
250 | for geom_type, coords in zip(*zip(*map(dict.values, geometry))):
251 | geom_func = getattr(shapely.geometry, geom_type)
252 | if 'Polygon' not in geom_type:
253 | geometry_collection.append(geom_func(coords))
254 | else:
255 | geometry_collection.append(geom_func(pt for pts in coords for pt in pts))
256 |
257 | geome_data = shapely.geometry.GeometryCollection(geometry_collection)
258 | if to_wkt:
259 | geome_data = geome_data.wkt
260 | # else:
261 | # geome_data = shapely.geometry.GeometryCollection(geometry_collection).geoms
262 |
263 | else:
264 | geome_data = geometry.copy()
265 | geometries = geome_data['geometry']['geometries']
266 | geome_data.update(
267 | {'geometry': cls.transform_geometry_collection(geometries, mode=1, to_wkt=True)})
268 |
269 | return geome_data
270 |
271 | @classmethod
272 | def transform_geometry(cls, layer_data, layer_name):
273 | """
274 | Reformat the field of ``'geometry'`` into
275 | `shapely.geometry `_
276 | object.
277 |
278 | :param layer_data: dataframe of a specific layer of PBF data
279 | :type layer_data: pandas.DataFrame | pandas.Series
280 | :param layer_name: name (geometric type) of the PBF layer
281 | :type layer_name: str
282 | :return: (OSM feature with) reformatted geometry field
283 | :rtype: pandas.DataFrame | pandas.Series
284 |
285 | **Examples**::
286 |
287 | >>> from pydriosm.reader import Transformer
288 |
289 | >>> # An example of points layer data
290 | >>> lyr_name = 'points'
291 | >>> dat_ = {
292 | ... 'type': 'Feature',
293 | ... 'geometry': {
294 | ... 'type': 'Point',
295 | ... 'coordinates': [-0.5134241, 52.6555853]
296 | ... },
297 | ... 'properties': {
298 | ... 'osm_id': '488432',
299 | ... 'name': None,
300 | ... 'barrier': None,
301 | ... 'highway': None,
302 | ... 'ref': None,
303 | ... 'address': None,
304 | ... 'is_in': None,
305 | ... 'place': None,
306 | ... 'man_made': None,
307 | ... 'other_tags': '"odbl"=>"clean"'
308 | ... },
309 | ... 'id': 488432
310 | ... }
311 | >>> lyr_data = pd.DataFrame.from_dict(dat_, orient='index').T
312 |
313 | >>> geom_dat = Transformer.transform_geometry(layer_data=lyr_data, layer_name=lyr_name)
314 | >>> geom_dat
315 | 0 POINT (-0.5134241 52.6555853)
316 | Name: geometry, dtype: object
317 |
318 | .. seealso::
319 |
320 | - Examples for the method
321 | :meth:`PBFReadParse.read_pbf()`.
322 | """
323 |
324 | geom_col_name = 'geometry'
325 |
326 | if layer_name == 'other_relations':
327 | if isinstance(layer_data, pd.DataFrame): # geom_col_name in layer_data.columns:
328 | geometries = pd.DataFrame(list(layer_data[geom_col_name]))['geometries']
329 | geom_data = geometries.map(cls.transform_geometry_collection)
330 | geom_data.name = geom_col_name
331 | else:
332 | geom_data = layer_data.map(lambda x: cls.transform_geometry_collection(x, mode=2))
333 |
334 | else: # `layer_data` can be 'points', 'lines', 'multilinestrings' or 'multipolygons'
335 | if isinstance(layer_data, pd.DataFrame): # geom_col_name in layer_data.columns:
336 | geom_data = layer_data[geom_col_name].map(cls.transform_unitary_geometry)
337 | else:
338 | geom_data = layer_data.map(lambda x: cls.transform_unitary_geometry(x, mode=2))
339 |
340 | return geom_data
341 |
342 | @classmethod
343 | def transform_other_tags(cls, other_tags):
344 | """
345 | Reformat a ``'other_tags'`` from string into dictionary type.
346 |
347 | :param other_tags: data of ``'other_tags'`` of a single feature in a PBF data file
348 | :type other_tags: str | None
349 | :return: reformatted data of ``'other_tags'``
350 | :rtype: dict | None
351 |
352 | **Examples**::
353 |
354 | >>> from pydriosm.reader import Transformer
355 |
356 | >>> other_tags_dat = Transformer.transform_other_tags(other_tags='"odbl"=>"clean"')
357 | >>> other_tags_dat
358 | {'odbl': 'clean'}
359 |
360 | .. seealso::
361 |
362 | - Examples for the method
363 | :meth:`PBFReadParse.read_pbf()`.
364 | """
365 |
366 | if other_tags:
367 | tags = [re.sub(r'^"|"$', '', x) for x in re.split('(?<="),(?=")', other_tags)]
368 | try:
369 | fltr = (re.split(r'"=>"?', x, maxsplit=1) for x in filter(None, tags))
370 | except OtherTagsReformatError:
371 | fltr = filter(
372 | lambda x: len(x) == 2, (re.split(r'"=>"?', x) for x in filter(None, tags)))
373 | other_tags_ = {k: v.replace('
', ' ') for k, v in fltr}
374 |
375 | else: # e.g. the data of 'other_tags' is None
376 | other_tags_ = other_tags
377 |
378 | return other_tags_
379 |
380 | @classmethod
381 | def update_other_tags(cls, prop_or_feat, mode=1):
382 | """
383 | Update the original data of ``'other_tags'`` with parsed data.
384 |
385 | :param prop_or_feat: original data of a feature or a ``'properties'`` field
386 | :type prop_or_feat: dict
387 | :param mode: options include ``{1, 2}`` indicating what action to take;
388 | when ``mode=1`` (default), ``prop_or_feat`` should be data of a feature;
389 | when ``mode=2``, ``prop_or_feat`` should be data of a ``'properties'`` field
390 | :type mode: int
391 | :return: updated data of a feature or a 'properties' field
392 | :rtype: dict
393 |
394 | **Examples**::
395 |
396 | >>> from pydriosm.reader import Transformer
397 |
398 | >>> prop_dat = {
399 | ... 'properties': {
400 | ... 'osm_id': '488432',
401 | ... 'name': None,
402 | ... 'barrier': None,
403 | ... 'highway': None,
404 | ... 'ref': None,
405 | ... 'address': None,
406 | ... 'is_in': None,
407 | ... 'place': None,
408 | ... 'man_made': None,
409 | ... 'other_tags': '"odbl"=>"clean"'
410 | ... },
411 | ... }
412 |
413 | >>> prop_dat_ = Transformer.update_other_tags(prop_dat['properties'])
414 | >>> prop_dat_
415 | {'osm_id': '488432',
416 | 'name': None,
417 | 'barrier': None,
418 | 'highway': None,
419 | 'ref': None,
420 | 'address': None,
421 | 'is_in': None,
422 | 'place': None,
423 | 'man_made': None,
424 | 'other_tags': {'odbl': 'clean'}}
425 |
426 | .. seealso::
427 |
428 | - Examples for the method
429 | :meth:`PBFReadParse.read_pbf()`.
430 | """
431 |
432 | if mode == 1:
433 | properties = prop_or_feat.copy()
434 | properties.update({'other_tags': cls.transform_other_tags(properties['other_tags'])})
435 |
436 | return properties
437 |
438 | else:
439 | feat = prop_or_feat.copy()
440 | other_tags = copy.copy(feat['properties']['other_tags'])
441 |
442 | feat_ = update_dict(
443 | feat, {'properties': {'other_tags': cls.transform_other_tags(other_tags)}})
444 | # feat['properties'].update({'other_tags': transform_other_tags(other_tags)})
445 |
446 | return feat_
447 |
--------------------------------------------------------------------------------
/pydriosm/utils.py:
--------------------------------------------------------------------------------
1 | """
2 | Provide various helper functions for use across the package.
3 | """
4 |
5 | import importlib.resources
6 | import os
7 | import shutil
8 |
9 | from pyhelpers._cache import _check_dependency, _format_err_msg
10 | from pyhelpers.dirs import cd
11 |
12 |
13 | # ==================================================================================================
14 | # Data directories
15 | # ==================================================================================================
16 |
17 |
18 | def _cdd(*sub_dir, data_dir="data", mkdir=False, **kwargs):
19 | """
20 | Specify (or change to) a directory (or any subdirectories) for backup data of the package.
21 |
22 | :param sub_dir: [optional] name of a directory; names of directories (and/or a filename)
23 | :type sub_dir: str | os.PathLike[str]
24 | :param data_dir: name of a directory to store data, defaults to ``"data"``
25 | :type data_dir: str | os.PathLike[str]
26 | :param mkdir: whether to create a directory, defaults to ``False``
27 | :type mkdir: bool
28 | :param kwargs: [optional] parameters (e.g. ``mode=0o777``) of `os.makedirs`_
29 | :return: a full pathname of a directory or a file under the specified data directory ``data_dir``
30 | :rtype: str
31 |
32 | .. _`os.makedirs`: https://docs.python.org/3/library/os.html#os.makedirs
33 |
34 | **Example**::
35 |
36 | >>> from pydriosm.utils import _cdd
37 | >>> import os
38 |
39 | >>> path_to_dat_dir = _cdd(data_dir="data")
40 | >>> os.path.relpath(path_to_dat_dir)
41 | 'pydriosm\\data'
42 | """
43 |
44 | pathname = importlib.resources.files(__package__).joinpath(data_dir)
45 | for x in sub_dir:
46 | pathname = os.path.join(pathname, x)
47 |
48 | if mkdir:
49 | path_to_file, ext = os.path.splitext(pathname)
50 | if ext == '':
51 | os.makedirs(path_to_file, exist_ok=True, **kwargs)
52 | else:
53 | os.makedirs(os.path.dirname(pathname), exist_ok=True, **kwargs)
54 |
55 | return pathname
56 |
57 |
58 | def check_relpath(pathname, start=os.curdir):
59 | """
60 | Check and return a relative pathname to the given ``pathname``.
61 |
62 | On Windows, when ``pathname`` and ``start`` are on different drives, the function returns
63 | the given ``pathname``.
64 |
65 | :param pathname: pathname of a file or a directory
66 | :type pathname: str | os.PathLike[str]
67 | :param start: optional start directory,
68 | defaults to ``os.curdir`` (i.e. the current working directory)
69 | :type start: str | os.PathLike[str]
70 | :return: relative pathname to the given ``pathname``
71 | :type: str | os.PathLike[str]
72 | """
73 |
74 | try:
75 | relpath = os.path.relpath(pathname, start=start)
76 | except ValueError:
77 | relpath = pathname
78 |
79 | return relpath
80 |
81 |
82 | def cdd_geofabrik(*sub_dir, mkdir=False, default_dir="osm_geofabrik", **kwargs):
83 | """
84 | Change directory to ``osm_geofabrik\\`` and its subdirectories within a package.
85 |
86 | :param sub_dir: name of directory; names of directories (and/or a filename)
87 | :type sub_dir: str | os.PathLike
88 | :param mkdir: whether to create a directory, defaults to ``False``
89 | :type mkdir: bool
90 | :param default_dir: default folder name of the root directory for downloading data from Geofabrik,
91 | defaults to ``"osm_geofabrik"``
92 | :type default_dir: str
93 | :param kwargs: [optional] parameters of `pyhelpers.dir.cd()`_
94 | :return: an absolute path to a directory (or a file) under ``data_dir``
95 | :rtype: str | os.PathLike
96 |
97 | .. _`pyhelpers.dir.cd()`:
98 | https://pyhelpers.readthedocs.io/en/latest/_generated/pyhelpers.dir.cd.html
99 |
100 | **Examples**::
101 |
102 | >>> from pydriosm.utils import cdd_geofabrik
103 | >>> import os
104 |
105 | >>> os.path.relpath(cdd_geofabrik())
106 | 'osm_geofabrik'
107 | """
108 |
109 | pathname = cd(default_dir, *sub_dir, mkdir=mkdir, **kwargs)
110 |
111 | return pathname
112 |
113 |
114 | def cdd_bbbike(*sub_dir, mkdir=False, default_dir="osm_bbbike", **kwargs):
115 | """
116 | Change directory to ``osm_bbbike\\`` and its subdirectories.
117 |
118 | :param sub_dir: name of directory; names of directories (and/or a filename)
119 | :type sub_dir: str
120 | :param mkdir: whether to create a directory, defaults to ``False``
121 | :type mkdir: bool
122 | :param default_dir: default folder name of the root directory for downloading data from BBBike,
123 | defaults to ``"osm_bbbike"``
124 | :type default_dir: str
125 | :param kwargs: [optional] parameters of `pyhelpers.dir.cd()`_
126 | :return: an absolute path to a directory (or a file) under ``data_dir``
127 | :rtype: str
128 |
129 | .. _`pyhelpers.dir.cd()`:
130 | https://pyhelpers.readthedocs.io/en/latest/_generated/pyhelpers.dir.cd.html
131 |
132 | **Examples**::
133 |
134 | >>> from pydriosm.utils import cdd_bbbike
135 | >>> import os
136 |
137 | >>> os.path.relpath(cdd_bbbike())
138 | 'osm_bbbike'
139 | """
140 |
141 | pathname = cd(default_dir, *sub_dir, mkdir=mkdir, **kwargs)
142 |
143 | return pathname
144 |
145 |
146 | # ==================================================================================================
147 | # General utilities
148 | # ==================================================================================================
149 |
150 |
151 | def first_unique(iterable):
152 | """
153 | Return unique items in an input iterable variable given the same order of presence.
154 |
155 | :param iterable: iterable variable
156 | :type iterable: typing.Iterable
157 | :return: unique items in the same order of presence as in the input
158 | :rtype: typing.Generator[list]
159 |
160 | **Examples**::
161 |
162 | >>> from pydriosm.utils import first_unique
163 |
164 | >>> list_example1 = [1, 2, 2, 3, 4, 5, 6, 6, 2, 3, 1, 6]
165 | >>> list(first_unique(list_example1))
166 | [1, 2, 3, 4, 5, 6]
167 |
168 | >>> list_example2 = [6, 1, 2, 2, 3, 4, 5, 6, 6, 2, 3, 1]
169 | >>> list(first_unique(list_example2))
170 | [6, 1, 2, 3, 4, 5]
171 | """
172 |
173 | checked_list = []
174 |
175 | for x in iterable:
176 | if x not in checked_list:
177 | checked_list.append(x)
178 | yield x
179 |
180 |
181 | def check_json_engine(engine=None):
182 | """
183 | Check an available module used for loading JSON data.
184 |
185 | :param engine: name of a module for loading JSON data;
186 | when ``engine=None`` (default), use the built-in
187 | `json `_ module;
188 | :type engine: str | None
189 | :return: the module for loading JSON data
190 | :type: types.ModuleType | None
191 |
192 | **Examples**::
193 |
194 | >>> from pydriosm.utils import check_json_engine
195 | >>> import types
196 |
197 | >>> result = check_json_engine()
198 |
199 | >>> isinstance(result, types.ModuleType)
200 | True
201 | >>> result.__name__ == 'json'
202 | True
203 | """
204 |
205 | if engine is not None:
206 | valid_mod_names = {'ujson', 'orjson', 'rapidjson', 'json'}
207 | assert engine in valid_mod_names, f"`engine` must be on one of {valid_mod_names}."
208 | engine_ = _check_dependency(name=engine)
209 |
210 | else:
211 | engine_ = _check_dependency(name='json')
212 |
213 | return engine_
214 |
215 |
216 | def remove_osm_file(path_to_file, verbose=True):
217 | """
218 | Remove a downloaded OSM data file.
219 |
220 | :param path_to_file: absolute path to a downloaded OSM data file
221 | :type path_to_file: str
222 | :param verbose: defaults to ``True``
223 | :type verbose: bool
224 |
225 | **Examples**::
226 |
227 | >>> from pydriosm.utils import remove_osm_file
228 | >>> from pyhelpers.dirs import cd
229 | >>> import os
230 |
231 | >>> path_to_pseudo_pbf_file = cd('tests\\pseudo.osm.pbf')
232 | >>> try:
233 | ... open(path_to_pseudo_pbf_file, 'a').close()
234 | ... except OSError:
235 | ... print('Failed to create the file.')
236 | ... else:
237 | ... print('File created successfully.')
238 | File created successfully.
239 |
240 | >>> os.path.exists(path_to_pseudo_pbf_file)
241 | True
242 |
243 | >>> remove_osm_file(path_to_pseudo_pbf_file, verbose=True)
244 | Deleting "tests\\pseudo.osm.pbf" ... Done.
245 |
246 | >>> os.path.exists(path_to_pseudo_pbf_file)
247 | False
248 | """
249 |
250 | if not os.path.exists(path_to_file):
251 | if verbose:
252 | print("The file \"{}\" is not found at {}.".format(*os.path.split(path_to_file)[::-1]))
253 |
254 | else:
255 | if verbose:
256 | print(f"Deleting \"{check_relpath(path_to_file)}\"", end=" ... ")
257 |
258 | try:
259 | if os.path.isfile(path_to_file):
260 | os.remove(path_to_file)
261 | if verbose:
262 | print("Done.")
263 |
264 | elif os.path.isdir(path_to_file):
265 | shutil.rmtree(path_to_file)
266 | if verbose:
267 | print("Done.")
268 |
269 | except Exception as e:
270 | print(f"Failed. {_format_err_msg(e)}")
271 |
--------------------------------------------------------------------------------
/pyproject.toml:
--------------------------------------------------------------------------------
1 | [build-system]
2 | requires = [
3 | "setuptools>=67.0",
4 | "wheel"
5 | ]
6 | build-backend = "setuptools.build_meta"
--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | build==1.0.3
2 | fqdn==1.5.1
3 | gdal==3.4.3
4 | geopandas==0.14.0
5 | isoduration==20.11.0
6 | jsonpointer==2.4
7 | notebook==7.0.3
8 | orjson==3.9.7
9 | pip-chill==1.0.3
10 | pyrcs==0.3.7
11 | pyshp==2.3.1
12 | pytest-cov==4.1.0
13 | python-rapidjson==1.11
14 | sphinx-copybutton==0.5.2
15 | sphinx-rtd-theme==1.3.0
16 | tqdm==4.66.1
17 | twine==4.0.2
18 | ujson==5.8.0
19 | uri-template==1.3.0
20 | webcolors==1.13
21 |
--------------------------------------------------------------------------------
/setup.cfg:
--------------------------------------------------------------------------------
1 | [metadata]
2 | long_description = file: README.md
3 | long_description_content_type = text/markdown
4 | keywords =
5 | Python
6 | OpenStreetMap
7 | OSM
8 | Geofabrik
9 | BBBike
10 | Protocolbuffer Binary Format
11 | PBF
12 | Shapefile
13 | PostgreSQL
14 | platforms = nt, posix
15 | classifiers =
16 | Intended Audience :: Developers
17 | Intended Audience :: Education
18 | Intended Audience :: End Users/Desktop
19 | Intended Audience :: Information Technology
20 | Intended Audience :: Science/Research
21 | Topic :: Education
22 | Topic :: Scientific/Engineering
23 | Topic :: Software Development
24 | Topic :: Utilities
25 | License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
26 | Programming Language :: Python :: 3
27 | Operating System :: Microsoft :: Windows
28 | Operating System :: POSIX :: Linux
29 | license_files = LICENSE
30 |
31 | [options]
32 | packages =
33 | pydriosm
34 | pydriosm.data
35 | include_package_data = True
36 | python_requires = >=3.9
37 | install_requires =
38 | pyhelpers>=1.5.2
39 | pyrcs>=0.3.7
40 | pyshp>=2.3.1
41 |
42 | [options.package_data]
43 | * = data/*
44 |
45 | [options.packages.find]
46 | exclude =
47 | tests
48 | *.tests
49 | tests.*
50 |
--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
1 | import json
2 |
3 | import setuptools
4 |
5 | with open(file="pydriosm/data/metadata.json", mode='r') as metadata_file:
6 | metadata = json.load(metadata_file)
7 |
8 | __pkgname__, __version__ = metadata['Package'], metadata['Version']
9 |
10 | __home_page__ = 'https://github.com/mikeqfu/' + f'{__pkgname__}'
11 |
12 | setuptools.setup(
13 | name=__pkgname__,
14 | version=__version__,
15 | description=metadata['Description'],
16 | url=__home_page__,
17 | author=metadata['Author'],
18 | author_email=metadata['Email'],
19 | license=metadata['License'],
20 | project_urls={
21 | 'Documentation': f'https://{__pkgname__}.readthedocs.io/en/{__version__}/',
22 | 'Source': __home_page__,
23 | 'Issue Tracker': __home_page__ + '/issues',
24 | },
25 | )
26 |
--------------------------------------------------------------------------------
/tests/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/tests/__init__.py
--------------------------------------------------------------------------------
/tests/data/rutland/other_relations_1.pkl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/tests/data/rutland/other_relations_1.pkl
--------------------------------------------------------------------------------
/tests/data/rutland/other_relations_2.pkl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/tests/data/rutland/other_relations_2.pkl
--------------------------------------------------------------------------------
/tests/data/rutland/points_1.pkl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/tests/data/rutland/points_1.pkl
--------------------------------------------------------------------------------
/tests/data/rutland/points_2.pkl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/tests/data/rutland/points_2.pkl
--------------------------------------------------------------------------------
/tests/data/rutland/rutland-latest-free.shp.zip:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/tests/data/rutland/rutland-latest-free.shp.zip
--------------------------------------------------------------------------------
/tests/data/rutland/rutland-latest.osm.pbf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mikeqfu/pydriosm/da106fe1cc67587fe8f9bc7649aab2d94929d191/tests/data/rutland/rutland-latest.osm.pbf
--------------------------------------------------------------------------------
/tests/test_downlaoder.py:
--------------------------------------------------------------------------------
1 | """Test the module :py:mod:`pydriosm.downloader`."""
2 |
3 | import os
4 | import tempfile
5 |
6 | import pandas as pd
7 | import pytest
8 | from pyhelpers.dirs import delete_dir
9 |
10 | from pydriosm.downloader import BBBikeDownloader, GeofabrikDownloader
11 | from pydriosm.downloader._downloader import _Downloader
12 | from pydriosm.errors import InvalidFileFormatError, InvalidSubregionNameError
13 |
14 | gfd, bbd = GeofabrikDownloader(), BBBikeDownloader()
15 |
16 |
17 | class TestDownloader:
18 |
19 | @staticmethod
20 | def test_init():
21 | d = _Downloader()
22 |
23 | assert d.NAME == 'OSM Downloader'
24 | assert os.path.relpath(d.download_dir) == 'osm_data'
25 | assert os.path.relpath(d.cdd()) == 'osm_data'
26 | assert d.download_dir == d.cdd()
27 |
28 | d = _Downloader(download_dir="tests\\osm_data")
29 | assert os.path.relpath(d.download_dir) == 'tests\\osm_data'
30 |
31 | @staticmethod
32 | def test_cdd():
33 | assert os.path.relpath(_Downloader.cdd()) == 'osm_data'
34 |
35 | @staticmethod
36 | def test_compose_cfm_msg():
37 | assert _Downloader.compose_cfm_msg() == 'To compile data of \n?'
38 | assert _Downloader.compose_cfm_msg(update=True) == 'To update the data of \n?'
39 |
40 | @staticmethod
41 | def test_print_act_msg(capfd):
42 | assert _Downloader.print_act_msg(verbose=False) is None
43 |
44 | _Downloader.print_act_msg(verbose=True)
45 | print("Done.")
46 | out, _ = capfd.readouterr()
47 | assert out == 'Compiling the data ... Done.\n'
48 |
49 | _Downloader.print_act_msg(verbose=True, note="(Some notes here.)")
50 | print("Done.")
51 | out, _ = capfd.readouterr()
52 | assert out == 'Compiling the data (Some notes here.) ... Done.\n'
53 |
54 | _Downloader.print_act_msg(verbose=True, confirmation_required=False)
55 | print("Done.")
56 | out, _ = capfd.readouterr()
57 | assert out == 'Compiling data of ... Done.\n'
58 |
59 | @staticmethod
60 | def test_print_otw_msg(capfd):
61 | assert _Downloader.print_otw_msg() is None
62 |
63 | _Downloader.print_otw_msg(verbose=True)
64 | out, _ = capfd.readouterr()
65 | assert out == 'Cancelled.\n'
66 |
67 | _Downloader.print_otw_msg(verbose=2)
68 | out, _ = capfd.readouterr()
69 | assert out == 'The collecting of is cancelled, or no data is available.\n'
70 |
71 | _Downloader.print_otw_msg(verbose=True, error_message="Errors.")
72 | out, _ = capfd.readouterr()
73 | assert out == 'Failed. Errors.\n'
74 |
75 | @staticmethod
76 | @pytest.mark.parametrize('data_name', [None, ''])
77 | def test_get_prepacked_data(data_name, monkeypatch, capfd):
78 | rslt = _Downloader.get_prepacked_data(
79 | callable, data_name=data_name, confirmation_required=False)
80 | assert rslt is None
81 |
82 | monkeypatch.setattr('builtins.input', lambda _: "No")
83 | rslt = _Downloader.get_prepacked_data(callable, verbose=True)
84 | out, _ = capfd.readouterr()
85 | assert 'Cancelled.' in out
86 | assert rslt is None
87 |
88 | @staticmethod
89 | def test_validate_subregion_name():
90 | with pytest.raises(InvalidSubregionNameError) as e:
91 | _Downloader.validate_subregion_name('abc')
92 | assert '1)' in e and '2)' in e
93 |
94 | with pytest.raises(InvalidSubregionNameError) as e:
95 | _Downloader.validate_subregion_name('abc', ['ab'])
96 | assert ' -> ' in e
97 |
98 | subrgn_name = 'usa'
99 | subrgn_name_ = _Downloader.validate_subregion_name(subrgn_name)
100 | assert subrgn_name_ == 'United States of America'
101 |
102 | avail_subrgn_names = ['Greater London', 'Great Britain', 'Birmingham', 'Leeds']
103 |
104 | subrgn_name = 'Britain'
105 | subrgn_name_ = _Downloader.validate_subregion_name(subrgn_name, avail_subrgn_names)
106 | assert subrgn_name_ == 'Great Britain'
107 |
108 | subrgn_name = 'london'
109 | subrgn_name_ = _Downloader.validate_subregion_name(subrgn_name, avail_subrgn_names)
110 | assert subrgn_name_ == 'Greater London'
111 |
112 | @staticmethod
113 | def test_validate_file_format():
114 | with pytest.raises(InvalidFileFormatError) as e:
115 | file_fmt = 'abc'
116 | _Downloader.validate_file_format(file_fmt) # Raise an error
117 | assert "`osm_file_format='abc'` -> The input `osm_file_format` is unidentifiable." in e
118 |
119 | avail_file_fmts = ['.osm.pbf', '.shp.zip', '.osm.bz2']
120 |
121 | file_fmt = 'pbf'
122 | assert _Downloader.validate_file_format(file_fmt, avail_file_fmts) == '.osm.pbf'
123 |
124 | file_fmt = 'shp'
125 | assert _Downloader.validate_file_format(file_fmt, avail_file_fmts) == '.shp.zip'
126 |
127 | @staticmethod
128 | def test_get_default_sub_path():
129 | subrgn_name_ = 'London'
130 | dwnld_url = 'https://download.bbbike.org/osm/bbbike/London/London.osm.pbf'
131 |
132 | assert _Downloader.get_default_sub_path(subrgn_name_, dwnld_url) == '\\london'
133 |
134 | @staticmethod
135 | def test_make_subregion_dirname():
136 | subrgn_name_ = 'England'
137 | assert _Downloader.make_subregion_dirname(subrgn_name_) == 'england'
138 |
139 | subrgn_name_ = 'Greater London'
140 | assert _Downloader.make_subregion_dirname(subrgn_name_) == 'greater-london'
141 |
142 | @staticmethod
143 | def test_get_subregion_download_url():
144 | rslt = _Downloader.get_subregion_download_url('', '')
145 | assert rslt == ('', '')
146 |
147 | @staticmethod
148 | def test_get_valid_download_info():
149 | d = _Downloader()
150 |
151 | subregion_name, osm_file_format = 'subregion_name', 'osm_file_format'
152 |
153 | valid_dwnld_info = d.get_valid_download_info(subregion_name, osm_file_format)
154 | subregion_name_, osm_filename, download_url, file_pathname = valid_dwnld_info
155 | assert subregion_name_ == ''
156 | assert osm_filename == ''
157 | assert download_url == ''
158 | assert os.path.relpath(file_pathname) == 'osm_data\\\\'
159 |
160 | d.download_dir = os.path.join(d.download_dir, '')
161 | _, _, _, file_pathname = d.get_valid_download_info(subregion_name, osm_file_format)
162 | assert os.path.relpath(file_pathname) == 'osm_data\\\\'
163 |
164 | download_dir = 'x-osm-pbf'
165 | _, _, _, file_pathname = d.get_valid_download_info(
166 | subregion_name, osm_file_format, download_dir)
167 | assert os.path.relpath(file_pathname) == 'x-osm-pbf\\'
168 |
169 | subregion_name, osm_file_format = '', ''
170 | _, osm_filename, _, file_pathname = d.get_valid_download_info(
171 | subregion_name=subregion_name, osm_file_format=osm_file_format)
172 | assert osm_filename is None and file_pathname is None
173 |
174 | @staticmethod
175 | def test_file_exists(capfd):
176 | d = _Downloader()
177 |
178 | subregion_name = ''
179 | osm_file_format = 'shp'
180 | rslt = d.file_exists(subregion_name=subregion_name, osm_file_format=osm_file_format)
181 | assert not rslt
182 |
183 | subregion_name, osm_file_format = '', ''
184 | rslt = d.file_exists(
185 | subregion_name=subregion_name, osm_file_format=osm_file_format, verbose=2)
186 | out, _ = capfd.readouterr()
187 | assert 'None data for "None" is not available' in out
188 | assert not rslt
189 |
190 | @staticmethod
191 | def test_file_exists_and_more():
192 | subrgn_names, file_format = 'London', ".pbf"
193 |
194 | rslt = gfd.file_exists_and_more(subregion_names=subrgn_names, osm_file_format=file_format)
195 | assert rslt == (['Greater London'], '.osm.pbf', True, 'download', ['Greater London'], [])
196 |
197 | rslt = bbd.file_exists_and_more(subregion_names=subrgn_names, osm_file_format=file_format)
198 | assert rslt == (['London'], '.pbf', True, 'download', ['London'], [])
199 |
200 | subrgn_names = ['london', 'rutland']
201 | rslt = gfd.file_exists_and_more(subregion_names=subrgn_names, osm_file_format=file_format)
202 | assert rslt == (
203 | ['Greater London', 'Rutland'], '.osm.pbf', True, 'download',
204 | ['Greater London', 'Rutland'],
205 | [])
206 |
207 | subrgn_names = ['birmingham', 'leeds']
208 | rslt = bbd.file_exists_and_more(subregion_names=subrgn_names, osm_file_format=file_format)
209 | assert rslt == (
210 | ['Birmingham', 'Leeds'], '.pbf', True, 'download', ['Birmingham', 'Leeds'], [])
211 |
212 | @staticmethod
213 | def test_verify_download_dir():
214 | d = _Downloader()
215 | assert os.path.relpath(d.download_dir) == 'osm_data'
216 |
217 | d.verify_download_dir(download_dir='tests\\osm_data', verify_download_dir=True)
218 | assert os.path.relpath(d.download_dir) == 'tests\\osm_data'
219 |
220 |
221 | class TestGeofabrikDownloader:
222 |
223 | @staticmethod
224 | def test_init():
225 | assert gfd.NAME == 'Geofabrik'
226 | assert gfd.URL == 'https://download.geofabrik.de/'
227 | assert gfd.DOWNLOAD_INDEX_URL == 'https://download.geofabrik.de/index-v1.json'
228 | assert os.path.relpath(gfd.download_dir) == 'osm_data\\geofabrik'
229 |
230 | gfd_ = GeofabrikDownloader(download_dir="tests\\osm_data")
231 | assert os.path.relpath(gfd_.download_dir) == 'tests\\osm_data'
232 |
233 | assert isinstance(gfd.valid_subregion_names, set)
234 | assert isinstance(gfd.download_index, pd.DataFrame)
235 | assert isinstance(gfd.continent_tables, dict)
236 | assert isinstance(gfd.region_subregion_tier, dict)
237 | assert isinstance(gfd.having_no_subregions, list)
238 | assert isinstance(gfd.catalogue, pd.DataFrame)
239 |
240 | @staticmethod
241 | def test_get_raw_directory_index(capfd):
242 | raw_index = gfd.get_raw_directory_index(gfd.URL, verbose=True)
243 | out, _ = capfd.readouterr()
244 | assert out == \
245 | "Collecting the raw directory index on 'https://download.geofabrik.de/' ... " \
246 | "Failed.\n" \
247 | "No raw directory index is available on the web page.\n"
248 | assert raw_index is None
249 |
250 | great_britain_url = 'https://download.geofabrik.de/europe/great-britain.html'
251 | raw_index = gfd.get_raw_directory_index(great_britain_url)
252 | assert isinstance(raw_index, pd.DataFrame)
253 | assert raw_index.columns.to_list() == ['file', 'date', 'size', 'metric_file_size', 'url']
254 |
255 | @staticmethod
256 | def test_get_download_index():
257 | geofabrik_dwnld_idx = gfd.get_download_index()
258 |
259 | assert isinstance(geofabrik_dwnld_idx, pd.DataFrame)
260 | assert geofabrik_dwnld_idx.columns.to_list() == [
261 | 'id',
262 | 'parent',
263 | 'iso3166-1:alpha2',
264 | 'name',
265 | 'iso3166-2',
266 | 'geometry',
267 | '.osm.pbf',
268 | '.osm.bz2',
269 | '.shp.zip',
270 | 'pbf-internal',
271 | 'history',
272 | 'taginfo',
273 | 'updates']
274 |
275 | @staticmethod
276 | def test_get_subregion_table(capfd):
277 | homepage = gfd.get_subregion_table(url=gfd.URL)
278 |
279 | assert homepage.columns.to_list() == [
280 | 'subregion',
281 | 'subregion-url',
282 | '.osm.pbf',
283 | '.osm.pbf-size',
284 | '.shp.zip',
285 | '.osm.bz2']
286 |
287 | great_britain_url = 'https://download.geofabrik.de/europe/great-britain.html'
288 | great_britain = gfd.get_subregion_table(great_britain_url)
289 | assert isinstance(great_britain, pd.DataFrame)
290 |
291 | antarctica_url = 'https://download.geofabrik.de/antarctica.html'
292 | antarctica = gfd.get_subregion_table(antarctica_url, verbose=True)
293 | out, _ = capfd.readouterr()
294 | assert out == 'Compiling information about subregions of "Antarctica" ... Failed.\n'
295 | assert antarctica is None
296 |
297 | antarctica2 = gfd.get_subregion_table(antarctica_url, verbose=2)
298 | out, _ = capfd.readouterr()
299 | assert out == \
300 | "Compiling information about subregions of \"Antarctica\" ... Failed.\n" \
301 | "No subregion data is available for \"Antarctica\" on " \
302 | "Geofabrik's free download server.\n"
303 | assert antarctica2 is None
304 |
305 | @staticmethod
306 | def test_get_continent_tables():
307 | continent_tables = gfd.get_continent_tables()
308 | assert isinstance(continent_tables, dict)
309 |
310 | asia_table = continent_tables['Asia']
311 | assert asia_table.columns.to_list() == [
312 | 'subregion',
313 | 'subregion-url',
314 | '.osm.pbf',
315 | '.osm.pbf-size',
316 | '.shp.zip',
317 | '.osm.bz2']
318 |
319 | @staticmethod
320 | def test_get_region_subregion_tier():
321 | rgn_subrgn_tier, no_subrgn_list = gfd.get_region_subregion_tier()
322 | assert isinstance(rgn_subrgn_tier, dict)
323 | assert isinstance(no_subrgn_list, list)
324 |
325 | @staticmethod
326 | def test_get_catalogue():
327 | dwnld_catalog = gfd.get_catalogue()
328 | assert isinstance(dwnld_catalog, pd.DataFrame)
329 | assert len(dwnld_catalog) >= 474
330 | assert dwnld_catalog.columns.to_list() == [
331 | 'subregion',
332 | 'subregion-url',
333 | '.osm.pbf',
334 | '.osm.pbf-size',
335 | '.shp.zip',
336 | '.osm.bz2']
337 |
338 | @staticmethod
339 | def test_get_valid_subregion_names():
340 | valid_subrgn_names = gfd.get_valid_subregion_names()
341 | assert isinstance(valid_subrgn_names, set)
342 |
343 | @staticmethod
344 | def test_validate_subregion_name():
345 | input_subrgn_name = 'london'
346 | valid_subrgn_name = gfd.validate_subregion_name(subregion_name=input_subrgn_name)
347 | assert valid_subrgn_name == 'Greater London'
348 |
349 | input_subrgn_name = 'https://download.geofabrik.de/europe/great-britain.html'
350 | valid_subrgn_name = gfd.validate_subregion_name(subregion_name=input_subrgn_name)
351 | assert valid_subrgn_name == 'Great Britain'
352 |
353 | @staticmethod
354 | def test_validate_file_format():
355 | input_file_format = ".pbf"
356 | valid_file_format = gfd.validate_file_format(osm_file_format=input_file_format)
357 | assert valid_file_format == '.osm.pbf'
358 |
359 | input_file_format = "shp"
360 | valid_file_format = gfd.validate_file_format(osm_file_format=input_file_format)
361 | assert valid_file_format == '.shp.zip'
362 |
363 | @staticmethod
364 | def test_get_subregion_download_url():
365 | subrgn_name = 'England'
366 | file_format = ".pbf"
367 | valid_name, dwnld_link = gfd.get_subregion_download_url(subrgn_name, file_format)
368 | assert valid_name == 'England'
369 | assert dwnld_link == \
370 | 'https://download.geofabrik.de/europe/great-britain/england-latest.osm.pbf'
371 |
372 | subrgn_name = 'britain'
373 | file_format = ".shp"
374 | valid_name, dwnld_link = gfd.get_subregion_download_url(subrgn_name, file_format)
375 | assert valid_name == 'Great Britain'
376 | assert dwnld_link is None
377 |
378 | @staticmethod
379 | def test_get_default_filename(capfd):
380 | subrgn_name, file_format = 'london', ".pbf"
381 | default_fn = gfd.get_default_filename(subrgn_name, file_format)
382 | assert default_fn == 'greater-london-latest.osm.pbf'
383 |
384 | subrgn_name, file_format = 'britain', ".shp"
385 | default_fn = gfd.get_default_filename(subrgn_name, file_format)
386 | out, _ = capfd.readouterr()
387 | assert out == 'No .shp.zip data is available to download for Great Britain.\n'
388 | assert default_fn is None
389 |
390 | @staticmethod
391 | def test_get_default_pathname():
392 | subrgn_name, file_format = 'london', ".pbf"
393 |
394 | pathname, filename = gfd.get_default_pathname(subrgn_name, file_format)
395 | assert os.path.relpath(os.path.dirname(pathname)) == \
396 | 'osm_data\\geofabrik\\europe\\great-britain\\england\\greater-london'
397 | assert filename == 'greater-london-latest.osm.pbf'
398 |
399 | @staticmethod
400 | def test_get_subregions():
401 | all_subrgn_names = gfd.get_subregions()
402 | assert isinstance(all_subrgn_names, list)
403 |
404 | e_na_subrgn_names = gfd.get_subregions('england', 'n america')
405 | assert isinstance(e_na_subrgn_names, list)
406 |
407 | na_subrgn_names = gfd.get_subregions('n america', deep=True)
408 | assert isinstance(na_subrgn_names, list)
409 |
410 | gb_subrgn_names = gfd.get_subregions('britain')
411 | gb_subrgn_names_ = gfd.get_subregions('britain', deep=True)
412 | assert len(gb_subrgn_names_) >= len(gb_subrgn_names)
413 |
414 | @staticmethod
415 | def test_specify_sub_download_dir():
416 | subrgn_name = 'london'
417 | file_format = ".pbf"
418 |
419 | dwnld_dir = gfd.specify_sub_download_dir(subrgn_name, file_format)
420 | assert os.path.dirname(os.path.relpath(dwnld_dir)) == \
421 | 'osm_data\\geofabrik\\europe\\great-britain\\england\\greater-london'
422 |
423 | dwnld_dir = "tests\\osm_data"
424 |
425 | subrgn_name = 'britain'
426 | file_format = ".shp"
427 |
428 | dwnld_pathname = gfd.specify_sub_download_dir(subrgn_name, file_format, dwnld_dir)
429 | assert os.path.relpath(dwnld_pathname) == 'tests\\osm_data\\great-britain-shp-zip'
430 |
431 | gfd_ = GeofabrikDownloader(download_dir=dwnld_dir)
432 | dwnld_pathname_ = gfd_.specify_sub_download_dir(subrgn_name, file_format)
433 | assert os.path.relpath(dwnld_pathname_) == \
434 | 'tests\\osm_data\\europe\\great-britain\\great-britain-shp-zip'
435 |
436 | @staticmethod
437 | def test_get_valid_download_info():
438 | subrgn_name = 'london'
439 | file_format = "pbf"
440 |
441 | info1 = gfd.get_valid_download_info(subrgn_name, file_format)
442 | valid_subrgn_name, pbf_filename, dwnld_url, path_to_pbf = info1
443 |
444 | assert valid_subrgn_name == 'Greater London'
445 | assert pbf_filename == 'greater-london-latest.osm.pbf'
446 | assert dwnld_url == \
447 | 'https://download.geofabrik.de/europe/great-britain/england/' \
448 | 'greater-london-latest.osm.pbf'
449 | assert os.path.relpath(path_to_pbf) == \
450 | 'osm_data\\geofabrik\\europe\\great-britain\\england\\greater-london\\' \
451 | 'greater-london-latest.osm.pbf'
452 |
453 | dwnld_dir = "tests\\osm_data"
454 |
455 | info2 = gfd.get_valid_download_info(subrgn_name, file_format, dwnld_dir)
456 | _, _, _, path_to_pbf2 = info2
457 |
458 | assert os.path.relpath(path_to_pbf2) == \
459 | 'tests\\osm_data\\greater-london\\greater-london-latest.osm.pbf'
460 |
461 | gfd_ = GeofabrikDownloader(download_dir=dwnld_dir)
462 |
463 | info3 = gfd_.get_valid_download_info(subrgn_name, file_format)
464 | _, _, _, path_to_pbf3 = info3
465 |
466 | assert os.path.relpath(path_to_pbf3) == \
467 | 'tests\\osm_data\\europe\\great-britain\\england\\greater-london\\' \
468 | 'greater-london-latest.osm.pbf'
469 |
470 | @staticmethod
471 | def test_download_osm_data(capfd):
472 | gfd_ = GeofabrikDownloader()
473 |
474 | subrgn_names = ['london', 'rutland']
475 | file_format = ".pbf"
476 | gfd_.download_osm_data(
477 | subregion_names=subrgn_names, osm_file_format=file_format, verbose=True,
478 | confirmation_required=False)
479 | out, _ = capfd.readouterr()
480 | assert "Downloading " in out and "Done." in out
481 | assert len(gfd_.data_paths) == 2
482 | assert os.path.relpath(gfd_.download_dir) == 'osm_data\\geofabrik'
483 |
484 | dwnld_dir = os.path.dirname(gfd_.download_dir)
485 |
486 | region_name = 'west midlands'
487 | file_format = ".shp"
488 | temp_dwnld_dir = tempfile.TemporaryDirectory().name
489 |
490 | gfd_.download_osm_data(
491 | subregion_names=region_name, osm_file_format=file_format, download_dir=temp_dwnld_dir,
492 | confirmation_required=False)
493 | assert len(gfd_.data_paths) == 3
494 | assert gfd_.data_paths[-1] == \
495 | f'{temp_dwnld_dir}\\west-midlands\\west-midlands-latest-free.shp.zip'
496 | assert gfd_.download_dir == temp_dwnld_dir
497 | assert os.path.relpath(gfd_.cdd()) == 'osm_data\\geofabrik'
498 |
499 | delete_dir([dwnld_dir, temp_dwnld_dir], confirmation_required=False)
500 |
501 | # @staticmethod
502 | # def test_download_subregion_data():
503 | # gfd_ = GeofabrikDownloader()
504 | #
505 | # subrgn_name = 'England'
506 | # file_format = ".pbf"
507 | # dwnld_dir = "tests\\osm_data"
508 | #
509 | # dwnld_file_pathnames = gfd_.download_subregion_data(
510 | # subrgn_name, file_format, download_dir=dwnld_dir, update=True, verbose=True,
511 | # ret_download_path=True, confirmation_required=False)
512 | #
513 | # assert len(dwnld_file_pathnames) >= 47
514 | # assert os.path.commonpath(dwnld_file_pathnames) == gfd_.download_dir
515 | #
516 | # delete_dir(gfd_.download_dir, confirmation_required=False)
517 |
518 |
519 | class TestBBBikeDownloader:
520 |
521 | @staticmethod
522 | def test_init():
523 | assert bbd.NAME == 'BBBike'
524 | assert bbd.LONG_NAME == 'BBBike exports of OpenStreetMap data'
525 | assert bbd.URL == 'https://download.bbbike.org/osm/bbbike/'
526 | assert os.path.relpath(bbd.download_dir) == 'osm_data\\bbbike'
527 |
528 | bbd_ = BBBikeDownloader(download_dir="tests\\osm_data")
529 | assert os.path.relpath(bbd_.download_dir) == 'tests\\osm_data'
530 |
531 | assert isinstance(bbd.valid_subregion_names, list)
532 | assert isinstance(bbd.subregion_coordinates, pd.DataFrame)
533 | assert isinstance(bbd.subregion_index, pd.DataFrame)
534 | assert isinstance(bbd.catalogue, dict)
535 |
536 | @staticmethod
537 | def test_get_names_of_cities():
538 | bbbike_cities = bbd.get_names_of_cities()
539 | assert isinstance(bbbike_cities, list)
540 |
541 | @staticmethod
542 | def test_get_coordinates_of_cities():
543 | coords_of_cities = bbd.get_coordinates_of_cities()
544 |
545 | assert isinstance(coords_of_cities, pd.DataFrame)
546 | assert coords_of_cities.columns.to_list() == [
547 | 'city',
548 | 'real_name',
549 | 'pref._language',
550 | 'local_language',
551 | 'country',
552 | 'area_or_continent',
553 | 'population',
554 | 'step',
555 | 'other_cities',
556 | 'll_longitude',
557 | 'll_latitude',
558 | 'ur_longitude',
559 | 'ur_latitude']
560 |
561 | @staticmethod
562 | def test_get_subregion_index():
563 | subrgn_idx = bbd.get_subregion_index()
564 |
565 | assert isinstance(subrgn_idx, pd.DataFrame)
566 | assert subrgn_idx.columns.to_list() == ['name', 'last_modified', 'url']
567 |
568 | @staticmethod
569 | def test_get_valid_subregion_names():
570 | subrgn_names = bbd.get_valid_subregion_names()
571 |
572 | assert isinstance(subrgn_names, list)
573 |
574 | @staticmethod
575 | def test_validate_subregion_name():
576 | subrgn_name = 'birmingham'
577 |
578 | valid_name = bbd.validate_subregion_name(subregion_name=subrgn_name)
579 | assert valid_name == 'Birmingham'
580 |
581 | @staticmethod
582 | def test_get_subregion_catalogue(capfd):
583 | subrgn_name = 'birmingham'
584 |
585 | bham_dwnld_cat = bbd.get_subregion_catalogue(
586 | subregion_name=subrgn_name, confirmation_required=False, verbose=True)
587 | out, _ = capfd.readouterr()
588 | assert 'Compiling the data of a download catalogue for "Birmingham" ... Done.\n' == out
589 | assert isinstance(bham_dwnld_cat, pd.DataFrame)
590 | assert bham_dwnld_cat.columns.to_list() == [
591 | 'filename', 'url', 'data_type', 'size', 'last_update']
592 |
593 | @staticmethod
594 | def test_get_catalogue():
595 | bbbike_catalogue = bbd.get_catalogue()
596 | assert list(bbbike_catalogue.keys()) == ['FileFormat', 'DataType', 'Catalogue']
597 |
598 | catalogue = bbbike_catalogue['Catalogue']
599 | assert isinstance(catalogue, dict)
600 |
601 | bham_catalogue = catalogue['Birmingham']
602 | assert isinstance(bham_catalogue, pd.DataFrame)
603 |
604 | @staticmethod
605 | def test_validate_file_format():
606 | valid_file_format = bbd.validate_file_format(osm_file_format='PBF')
607 | assert valid_file_format == '.pbf'
608 |
609 | valid_file_format = bbd.validate_file_format(osm_file_format='.osm.pbf')
610 | assert valid_file_format == '.pbf'
611 |
612 | @staticmethod
613 | def test_get_subregion_download_url():
614 | subrgn_name = 'birmingham'
615 | file_format = "pbf"
616 |
617 | subrgn_name_, dwnld_url = bbd.get_subregion_download_url(subrgn_name, file_format)
618 | assert subrgn_name_ == 'Birmingham'
619 | assert dwnld_url == 'https://download.bbbike.org/osm/bbbike/Birmingham/Birmingham.osm.pbf'
620 |
621 | file_format = "csv.xz"
622 | subrgn_name_, dwnld_url = bbd.get_subregion_download_url(subrgn_name, file_format)
623 |
624 | assert subrgn_name_ == 'Birmingham'
625 | assert dwnld_url == \
626 | 'https://download.bbbike.org/osm/bbbike/Birmingham/Birmingham.osm.csv.xz'
627 |
628 | @staticmethod
629 | def test_get_valid_download_info():
630 | subrgn_name = 'birmingham'
631 | file_format = "pbf"
632 |
633 | info = bbd.get_valid_download_info(subrgn_name, file_format)
634 | valid_subrgn_name, pbf_filename, dwnld_url, pbf_pathname = info
635 |
636 | assert valid_subrgn_name == 'Birmingham'
637 | assert pbf_filename == 'Birmingham.osm.pbf'
638 | assert dwnld_url == 'https://download.bbbike.org/osm/bbbike/Birmingham/Birmingham.osm.pbf'
639 | assert os.path.relpath(pbf_pathname) == 'osm_data\\bbbike\\birmingham\\Birmingham.osm.pbf'
640 |
641 | bbd_ = BBBikeDownloader(download_dir="tests\\osm_data")
642 | _, _, _, pbf_pathname = bbd_.get_valid_download_info(subrgn_name, file_format)
643 | assert os.path.relpath(pbf_pathname) == 'tests\\osm_data\\birmingham\\Birmingham.osm.pbf'
644 |
645 | @staticmethod
646 | def test_file_exists():
647 | subrgn_name = 'birmingham'
648 | file_format = ".pbf"
649 | dwnld_dir = "tests\\osm_data"
650 |
651 | pbf_exists = bbd.file_exists(subrgn_name, file_format, dwnld_dir)
652 | assert not pbf_exists
653 |
654 | # @staticmethod
655 | # def test_download_subregion_data(capfd):
656 | # subrgn_name = 'leeds'
657 | # dwnld_dir = "tests\\osm_data"
658 | #
659 | # bbd.download_subregion_data(
660 | # subregion_name=subrgn_name, download_dir=dwnld_dir, ret_download_path=True,
661 | # verbose=True, confirmation_required=False)
662 | # out, _ = capfd.readouterr()
663 | # assert 'Check out the downloaded OSM data at "tests\\osm_data\\leeds\\".' in out
664 |
665 | @staticmethod
666 | def test_download_osm_data():
667 | subrgn_name = 'London'
668 | file_format = "pbf"
669 |
670 | bbd.download_osm_data(subrgn_name, file_format, confirmation_required=False)
671 | assert len(bbd.data_paths) == 1
672 | assert os.path.relpath(bbd.data_paths[0]) == 'osm_data\\bbbike\\london\\London.osm.pbf'
673 |
674 | london_dwnld_dir = os.path.relpath(bbd.download_dir)
675 | assert london_dwnld_dir == 'osm_data\\bbbike'
676 |
677 | subrgn_names = ['leeds', 'birmingham']
678 | dwnld_dir = "tests\\osm_data"
679 |
680 | dwnld_paths = bbd.download_osm_data(
681 | subrgn_names, file_format, dwnld_dir, confirmation_required=False,
682 | ret_download_path=True)
683 | assert len(dwnld_paths) == 2
684 | assert len(bbd.data_paths) == 3
685 | assert os.path.relpath(bbd.download_dir) == os.path.relpath(dwnld_dir)
686 | assert os.path.relpath(os.path.commonpath(dwnld_paths)) == 'tests\\osm_data'
687 |
688 | delete_dir([os.path.dirname(london_dwnld_dir), dwnld_dir], confirmation_required=False)
689 |
690 |
691 | if __name__ == '__main__':
692 | pytest.main()
693 |
--------------------------------------------------------------------------------
/tests/test_errors.py:
--------------------------------------------------------------------------------
1 | """Test the module :py:mod:`pydriosm.errors`."""
2 |
3 | import pytest
4 |
5 |
6 | class TestInvalidSubregionNameError:
7 |
8 | @staticmethod
9 | def test_error():
10 | from pydriosm.errors import InvalidSubregionNameError
11 |
12 | msg = InvalidSubregionNameError('abc').message
13 | assert 'The input of `subregion_name` is not recognizable.' in msg
14 |
15 | msg = InvalidSubregionNameError('abc').__str__()
16 | assert ' -> ' in msg
17 |
18 | msg = InvalidSubregionNameError('abc', msg=1).message
19 | assert '1)' in msg and '2)' in msg
20 |
21 |
22 | class TestInvalidFileFormatError:
23 |
24 | @staticmethod
25 | def test_error():
26 | from pydriosm.errors import InvalidFileFormatError
27 |
28 | msg = InvalidFileFormatError('abc').message
29 | assert 'The input `osm_file_format` is unidentifiable.' in msg
30 |
31 | msg = InvalidFileFormatError('abc').__str__()
32 | assert ' -> ' in msg
33 |
34 | msg = InvalidFileFormatError('abc', valid_file_formats={'valid_file_formats'}).message
35 | assert 'Valid options include:' in msg
36 |
37 |
38 | class TestOtherTagsReformatError:
39 |
40 | @staticmethod
41 | def test_error():
42 | from pydriosm.errors import OtherTagsReformatError
43 |
44 | msg = OtherTagsReformatError('abc').message
45 | assert 'Failed to reformat the `other_tags`.' in msg
46 |
47 | msg = OtherTagsReformatError('abc').__str__()
48 | assert ' -> ' in msg
49 |
50 |
51 | if __name__ == '__main__':
52 | pytest.main()
53 |
--------------------------------------------------------------------------------
/tests/test_ios.py:
--------------------------------------------------------------------------------
1 | """Test the module :py:mod:`pydriosm.ios`."""
2 |
3 | import pytest
4 |
5 |
6 | def test_get_default_layer_name():
7 | from pydriosm.ios.utils import get_default_layer_name
8 |
9 | lyr_name = get_default_layer_name(schema_name='point')
10 | assert lyr_name == 'points'
11 |
12 | lyr_name = get_default_layer_name(schema_name='land')
13 | assert lyr_name == 'landuse'
14 |
15 |
16 | def test_validate_schema_names():
17 | from pydriosm.ios.utils import validate_schema_names
18 |
19 | valid_names = validate_schema_names()
20 | assert valid_names == []
21 |
22 | input_schema_names = ['point', 'polygon']
23 | valid_names = validate_schema_names(input_schema_names)
24 | assert valid_names == ['point', 'polygon']
25 |
26 | valid_names = validate_schema_names(input_schema_names, schema_named_as_layer=True)
27 | assert valid_names == ['points', 'multipolygons']
28 |
29 |
30 | def test_validate_table_name():
31 | from pydriosm.ios.utils import validate_table_name
32 |
33 | subrgn_name = 'greater london'
34 | valid_table_name = validate_table_name(subrgn_name)
35 | assert valid_table_name == 'greater london'
36 |
37 | subrgn_name = 'Llanfairpwllgwyngyllgogerychwyrndrobwllllantysiliogogogoch, Wales'
38 | valid_table_name = validate_table_name(subrgn_name, sub_space='_')
39 | assert valid_table_name == 'Llanfairpwllgwyngyllgogerychwyrndrobwllllantysiliogogogoch_W..'
40 |
41 |
42 | # from pydriosm.ios import PostgresOSM
43 | # from pydriosm.downloader import GeofabrikDownloader, BBBikeDownloader
44 | # from pydriosm.reader import GeofabrikReader, BBBikeReader
45 | #
46 | #
47 | # osmdb = PostgresOSM(database_name='osmdb_test')
48 | #
49 | #
50 | # class TestPostgresOSM:
51 | #
52 | # @staticmethod
53 | # def test_init():
54 | # assert osmdb.data_source == 'Geofabrik'
55 | # assert osmdb.name == 'Geofabrik OpenStreetMap data extracts'
56 | # assert osmdb.url == 'https://download.geofabrik.de/'
57 | # assert isinstance(osmdb.downloader, GeofabrikDownloader)
58 | # assert isinstance(osmdb.reader, GeofabrikReader)
59 | #
60 | # # Change the data source
61 | # assert osmdb.data_source == 'BBBike'
62 | # assert osmdb.name == 'BBBike exports of OpenStreetMap data'
63 | # assert osmdb.url == 'https://download.bbbike.org/osm/bbbike/'
64 | # assert isinstance(osmdb.downloader, BBBikeDownloader)
65 | # assert isinstance(osmdb.reader, BBBikeReader)
66 |
67 |
68 | if __name__ == '__main__':
69 | pytest.main()
70 |
--------------------------------------------------------------------------------
/tests/test_reader.py:
--------------------------------------------------------------------------------
1 | """Test the module :py:mod:`pydriosm.reader`."""
2 |
3 | import glob
4 | import os
5 | import shutil
6 |
7 | import pandas as pd
8 | import pytest
9 | import shapely.geometry
10 | from pyhelpers.store import load_pickle
11 |
12 | from pydriosm.reader import PBFReadParse, SHPReadParse, Transformer
13 | from pydriosm.reader._reader import _Reader
14 |
15 |
16 | class TestTransformer:
17 | TEST_POINT_1 = {
18 | 'type': 'Point',
19 | 'coordinates': [-0.5134241, 52.6555853]
20 | }
21 |
22 | TEST_POINT_2 = {
23 | 'type': 'Feature',
24 | 'geometry': {
25 | 'type': 'Point',
26 | 'coordinates': [-0.5134241, 52.6555853]
27 | },
28 | 'properties': {
29 | 'osm_id': '488432',
30 | 'name': None,
31 | 'barrier': None,
32 | 'highway': None,
33 | 'ref': None,
34 | 'address': None,
35 | 'is_in': None,
36 | 'place': None,
37 | 'man_made': None,
38 | 'other_tags': '"odbl"=>"clean"'
39 | },
40 | 'id': 488432
41 | }
42 |
43 | TEST_COLLECTION_1 = {
44 | 'type': 'GeometryCollection',
45 | 'geometries': [
46 | {'type': 'Point', 'coordinates': [-0.5096176, 52.6605168]},
47 | {'type': 'Point', 'coordinates': [-0.5097337, 52.6605812]}
48 | ]
49 | }
50 |
51 | TEST_COLLECTION_2 = {
52 | 'type': 'Feature',
53 | 'geometry': {
54 | 'type': 'GeometryCollection',
55 | 'geometries': [
56 | {'type': 'Point', 'coordinates': [-0.5096176, 52.6605168]},
57 | {'type': 'Point', 'coordinates': [-0.5097337, 52.6605812]}]
58 | },
59 | 'properties': {
60 | 'osm_id': '256254',
61 | 'name': 'Fife Close',
62 | 'type': 'site',
63 | 'other_tags': '"naptan:StopAreaCode"=>"270G02701525"'
64 | },
65 | 'id': 256254
66 | }
67 |
68 | @staticmethod
69 | def test_point_as_polygon():
70 | geometry = {
71 | 'type': 'MultiPolygon',
72 | 'coordinates': [[[[-0.6920145, 52.6753268], [-0.6920145, 52.6753268]]]]
73 | }
74 | mp_coords = geometry['coordinates']
75 |
76 | mp_coords_ = Transformer.point_as_polygon(mp_coords)
77 | assert mp_coords_ == [
78 | [[[-0.6920145, 52.6753268],
79 | [-0.6920145, 52.6753268],
80 | [-0.6920145, 52.6753268]]]]
81 |
82 | @classmethod
83 | def test_transform_unitary_geometry(cls):
84 | g1_dat = cls.TEST_POINT_1.copy()
85 | g1_data = Transformer.transform_unitary_geometry(g1_dat)
86 | assert isinstance(g1_data, shapely.geometry.Point)
87 | assert g1_data.wkt == 'POINT (-0.5134241 52.6555853)'
88 |
89 | g2_dat = cls.TEST_POINT_2.copy()
90 | g2_data = Transformer.transform_unitary_geometry(g2_dat, mode=2)
91 |
92 | assert isinstance(g2_data, dict)
93 | assert list(g2_data.keys()) == ['type', 'geometry', 'properties', 'id']
94 | assert g2_data['geometry'] == 'POINT (-0.5134241 52.6555853)'
95 |
96 | @classmethod
97 | def test_transform_geometry_collection(cls):
98 | g1_dat_ = cls.TEST_COLLECTION_1.copy()
99 | g1_dat = g1_dat_['geometries']
100 | g1_data = Transformer.transform_geometry_collection(g1_dat)
101 | assert isinstance(g1_data, shapely.geometry.base.BaseGeometry)
102 | assert (g1_data.wkt ==
103 | 'GEOMETRYCOLLECTION (POINT (-0.5096176 52.6605168), POINT (-0.5097337 52.6605812))')
104 |
105 | g2_dat = cls.TEST_COLLECTION_2.copy()
106 | g2_data = Transformer.transform_geometry_collection(g2_dat, mode=2)
107 | assert isinstance(g2_data, dict)
108 | assert list(g2_data.keys()) == ['type', 'geometry', 'properties', 'id']
109 | assert g2_data['geometry'] == \
110 | 'GEOMETRYCOLLECTION (POINT (-0.5096176 52.6605168), POINT (-0.5097337 52.6605812))'
111 |
112 | @classmethod
113 | def test_transform_geometry(cls):
114 | lyr_name = 'points'
115 | dat_ = cls.TEST_POINT_2.copy()
116 |
117 | lyr_data = pd.DataFrame.from_dict(dat_, orient='index').T
118 |
119 | geom_dat = Transformer.transform_geometry(layer_data=lyr_data, layer_name=lyr_name)
120 | assert isinstance(geom_dat, pd.Series)
121 | assert geom_dat.values[0].wkt == 'POINT (-0.5134241 52.6555853)'
122 |
123 | @staticmethod
124 | def test_transform_other_tags():
125 | other_tags_dat = Transformer.transform_other_tags(other_tags='"odbl"=>"clean"')
126 | assert other_tags_dat == {'odbl': 'clean'}
127 |
128 | @staticmethod
129 | def test_update_other_tags():
130 | prop_dat = {
131 | 'properties': {
132 | 'osm_id': '488432',
133 | 'name': None,
134 | 'barrier': None,
135 | 'highway': None,
136 | 'ref': None,
137 | 'address': None,
138 | 'is_in': None,
139 | 'place': None,
140 | 'man_made': None,
141 | 'other_tags': '"odbl"=>"clean"'
142 | },
143 | }
144 | prop_dat_ = Transformer.update_other_tags(prop_dat['properties'])
145 | assert prop_dat_ == {
146 | 'osm_id': '488432',
147 | 'name': None,
148 | 'barrier': None,
149 | 'highway': None,
150 | 'ref': None,
151 | 'address': None,
152 | 'is_in': None,
153 | 'place': None,
154 | 'man_made': None,
155 | 'other_tags': {'odbl': 'clean'}
156 | }
157 |
158 |
159 | class TestPBFReadParse:
160 | path_to_osm_pbf = "tests\\data\\rutland\\rutland-latest.osm.pbf"
161 |
162 | @staticmethod
163 | def test_get_pbf_layer_geom_types():
164 | pbf_layer_geom_dict = PBFReadParse.get_pbf_layer_geom_types(shape_name=True)
165 | assert pbf_layer_geom_dict == {
166 | 'points': 'Point',
167 | 'lines': 'LineString',
168 | 'multilinestrings': 'MultiLineString',
169 | 'multipolygons': 'MultiPolygon',
170 | 'other_relations': 'GeometryCollection'}
171 |
172 | @staticmethod
173 | @pytest.mark.parametrize('layer_name', ['points', 'other_relations'])
174 | @pytest.mark.parametrize('dat_id', [1, 2])
175 | @pytest.mark.parametrize('parse_geometry', [True, False])
176 | @pytest.mark.parametrize('parse_properties', [True, False])
177 | @pytest.mark.parametrize('parse_other_tags', [True, False])
178 | def test_transform_pbf_layer_field(layer_name, dat_id, parse_geometry, parse_properties,
179 | parse_other_tags):
180 | layer_data = load_pickle(f"tests\\data\\rutland\\{layer_name}_{dat_id}.pkl")
181 | lyr_dat = PBFReadParse.transform_pbf_layer_field(layer_data=layer_data, layer_name=layer_name)
182 |
183 | assert isinstance(lyr_dat, (pd.Series, pd.DataFrame))
184 |
185 | @pytest.mark.parametrize('readable', [False, True])
186 | @pytest.mark.parametrize('expand', [False, True])
187 | @pytest.mark.parametrize('number_of_chunks', [None, 5])
188 | def test_read_pbf(self, readable, expand, number_of_chunks):
189 | rutland_pbf = PBFReadParse.read_pbf(
190 | pbf_pathname=self.path_to_osm_pbf,
191 | readable=readable,
192 | expand=expand,
193 | number_of_chunks=number_of_chunks)
194 |
195 | assert isinstance(rutland_pbf, dict)
196 | assert list(rutland_pbf.keys()) == [
197 | 'points', 'lines', 'multilinestrings', 'multipolygons', 'other_relations']
198 |
199 |
200 | class TestSHPReadParse:
201 | path_to_shp_zip = "tests\\data\\rutland\\rutland-latest-free.shp.zip"
202 | extract_to_dir = "tests\\data\\rutland\\temp"
203 |
204 | @staticmethod
205 | def test_validate_shp_layer_names():
206 | assert SHPReadParse.validate_shp_layer_names(None) == []
207 | assert SHPReadParse.validate_shp_layer_names('point') == ['points']
208 | assert SHPReadParse.validate_shp_layer_names(['point', 'land']) == ['points', 'landuse']
209 | assert len(SHPReadParse.validate_shp_layer_names('all')) >= 13
210 |
211 | @staticmethod
212 | def test_find_shp_layer_name():
213 | assert SHPReadParse.find_shp_layer_name("") is None
214 | assert SHPReadParse.find_shp_layer_name("gis_osm_railways_free_1.shp") == 'railways'
215 | assert SHPReadParse.find_shp_layer_name("gis_osm_transport_a_free_1.shp") == 'transport'
216 |
217 | def test_unzip_shp_zip(self):
218 | rutland_shp_dir = SHPReadParse.unzip_shp_zip(
219 | self.path_to_shp_zip, extract_to=self.extract_to_dir, layer_names='railways',
220 | verbose=True, ret_extract_dir=True)
221 | assert rutland_shp_dir == self.extract_to_dir
222 |
223 | lyr_names = ['railways', 'transport', 'traffic']
224 | dirs_of_layers = SHPReadParse.unzip_shp_zip(
225 | self.path_to_shp_zip, extract_to=self.extract_to_dir, layer_names=lyr_names,
226 | separate=True, verbose=2, ret_extract_dir=True)
227 | assert os.path.relpath(os.path.commonpath(dirs_of_layers)) == self.extract_to_dir
228 | assert all(x in lyr_names for x in map(os.path.basename, dirs_of_layers))
229 |
230 | rutland_shp_dir = SHPReadParse.unzip_shp_zip(
231 | self.path_to_shp_zip, extract_to=self.extract_to_dir, ret_extract_dir=True, verbose=True)
232 | layer_names = set(
233 | filter(None, map(SHPReadParse.find_shp_layer_name, os.listdir(rutland_shp_dir))))
234 | assert all(x in SHPReadParse.LAYER_NAMES for x in layer_names)
235 |
236 | def test_read_shp(self):
237 | rutland_shp_dir = SHPReadParse.unzip_shp_zip(
238 | self.path_to_shp_zip, extract_to=self.extract_to_dir, ret_extract_dir=True)
239 | path_to_railways_shp = glob.glob(os.path.join(rutland_shp_dir, "*railways*.shp"))[0]
240 |
241 | rutland_railways = SHPReadParse.read_shp(path_to_railways_shp)
242 | assert isinstance(rutland_railways, pd.DataFrame)
243 |
244 | rutland_railways = SHPReadParse.read_shp(path_to_railways_shp, emulate_gpd=True)
245 | assert isinstance(rutland_railways, pd.DataFrame)
246 |
247 | rutland_railways_ = SHPReadParse.read_shp(path_to_railways_shp, engine='geopandas')
248 |
249 | railways_data = [rutland_railways, rutland_railways_]
250 | geom1, geom2 = map(lambda x: x['geometry'].map(lambda y: y.wkt), railways_data)
251 | assert geom1.equals(geom2)
252 |
253 | def test_read_layer_shps(self):
254 | rutland_shp_dir = SHPReadParse.unzip_shp_zip(
255 | self.path_to_shp_zip, extract_to=self.extract_to_dir, ret_extract_dir=True)
256 | rutland_railways_shp_path = os.path.join(rutland_shp_dir, "gis_osm_railways_free_1.shp")
257 |
258 | london_railways_shp = SHPReadParse.read_layer_shps(shp_pathnames=rutland_railways_shp_path)
259 | assert isinstance(london_railways_shp, pd.DataFrame)
260 |
261 | railways_rail_shp, railways_rail_shp_path = SHPReadParse.read_layer_shps(
262 | rutland_railways_shp_path, feature_names='rail', save_feat_shp=True,
263 | ret_feat_shp_path=True)
264 | assert isinstance(railways_rail_shp, pd.DataFrame)
265 | assert all(os.path.isfile(x) for x in railways_rail_shp_path)
266 |
267 | shutil.rmtree(self.extract_to_dir)
268 |
269 |
270 | class TestReader:
271 |
272 | @staticmethod
273 | def test_init():
274 | r = _Reader()
275 |
276 | assert r.NAME == 'OSM Reader'
277 | assert isinstance(r.SHP, type) and r.SHP == SHPReadParse
278 |
279 | @staticmethod
280 | def test_cdd():
281 | assert os.path.relpath(_Reader.cdd()) == 'osm_data'
282 |
283 | @staticmethod
284 | def test_data_dir():
285 | from pydriosm.downloader import GeofabrikDownloader, BBBikeDownloader
286 |
287 | r = _Reader()
288 | assert os.path.relpath(r.data_dir) == 'osm_data'
289 |
290 | r = _Reader(downloader=GeofabrikDownloader)
291 | assert os.path.relpath(r.data_dir) == 'osm_data\\geofabrik'
292 |
293 | r = _Reader(downloader=BBBikeDownloader)
294 | assert os.path.relpath(r.data_dir) == 'osm_data\\bbbike'
295 |
296 | @staticmethod
297 | def test_data_paths():
298 | r = _Reader()
299 | assert r.data_paths == []
300 |
301 |
302 | if __name__ == '__main__':
303 | pytest.main()
304 |
--------------------------------------------------------------------------------
/tests/test_updater.py:
--------------------------------------------------------------------------------
1 | """Test the module :py:mod:`pydriosm._updater`."""
2 |
3 | import pytest
4 |
5 |
6 | def test__update_prepacked_data(monkeypatch, capfd):
7 | from pydriosm._updater import _update_prepacked_data
8 |
9 | monkeypatch.setattr('builtins.input', lambda _: "Yes")
10 | _update_prepacked_data(verbose=True)
11 | out, _ = capfd.readouterr()
12 | assert "Done." in out and "Update finished." in out and "Failed." not in out
13 |
14 |
15 | if __name__ == '__main__':
16 | pytest.main()
17 |
--------------------------------------------------------------------------------
/tests/test_utils.py:
--------------------------------------------------------------------------------
1 | """Test the module :py:mod:`pydriosm.utils`."""
2 |
3 | import os
4 | import shutil
5 | import types
6 |
7 | import pytest
8 |
9 |
10 | @pytest.mark.parametrize('mkdir', [False, True])
11 | def test__cdd(mkdir):
12 | from pydriosm.utils import _cdd
13 |
14 | data_dir = "pytest_pydriosm_data"
15 | pathname = _cdd(data_dir=data_dir, mkdir=mkdir)
16 | assert os.path.basename(pathname) == data_dir
17 | if mkdir:
18 | os.rmdir(pathname)
19 |
20 | data_dir = "pytest_pydriosm_data\\tests.ext"
21 | pathname = _cdd(data_dir=data_dir, mkdir=mkdir)
22 | assert os.path.basename(pathname) == "tests.ext"
23 | if mkdir:
24 | shutil.rmtree(os.path.dirname(pathname))
25 |
26 |
27 | def test_cdd_geofabrik():
28 | from pydriosm.utils import cdd_geofabrik
29 |
30 | assert os.path.relpath(cdd_geofabrik()) == 'osm_geofabrik'
31 |
32 |
33 | def test_cdd_bbbike():
34 | from pydriosm.utils import cdd_bbbike
35 |
36 | assert os.path.relpath(cdd_bbbike()) == 'osm_bbbike'
37 |
38 |
39 | def test_first_unique():
40 | from pydriosm.utils import first_unique
41 |
42 | list_example1 = [1, 2, 2, 3, 4, 5, 6, 6, 2, 3, 1, 6]
43 | assert list(first_unique(list_example1)) == [1, 2, 3, 4, 5, 6]
44 |
45 | list_example2 = [6, 1, 2, 2, 3, 4, 5, 6, 6, 2, 3, 1]
46 | assert list(first_unique(list_example2)) == [6, 1, 2, 3, 4, 5]
47 |
48 |
49 | @pytest.mark.parametrize('engine', [None, 'ujson', 'orjson', 'rapidjson', 'json'])
50 | def test_check_json_engine(engine):
51 | from pydriosm.utils import check_json_engine
52 |
53 | result = check_json_engine(engine)
54 |
55 | assert isinstance(result, types.ModuleType)
56 |
57 |
58 | def test_remove_osm_file(capfd):
59 | from pydriosm.utils import remove_osm_file
60 |
61 | path_to_pseudo_pbf_file = os.path.join("tests\\data\\pseudo\\pseudo.osm.pbf")
62 |
63 | remove_osm_file(path_to_pseudo_pbf_file, verbose=True)
64 | out, _ = capfd.readouterr()
65 | assert 'The file "pseudo.osm.pbf" is not found' in out
66 |
67 | pseudo_dir = os.path.dirname(path_to_pseudo_pbf_file)
68 | os.makedirs(pseudo_dir)
69 | f = open(path_to_pseudo_pbf_file, 'w+')
70 |
71 | with pytest.raises(Exception) as e:
72 | assert os.path.exists(path_to_pseudo_pbf_file)
73 | remove_osm_file(path_to_pseudo_pbf_file)
74 | assert "Failed." in str(e.value)
75 |
76 | f.close()
77 |
78 | remove_osm_file(path_to_pseudo_pbf_file, verbose=True)
79 | out, _ = capfd.readouterr()
80 | assert "Deleting" in out and path_to_pseudo_pbf_file in out
81 | assert not os.path.exists(path_to_pseudo_pbf_file)
82 |
83 | remove_osm_file(pseudo_dir, verbose=True)
84 | assert "Deleting" in out and pseudo_dir in out
85 | assert not os.path.exists(pseudo_dir)
86 |
87 |
88 | if __name__ == '__main__':
89 | pytest.main()
90 |
--------------------------------------------------------------------------------