├── .coveragerc ├── .github └── workflows │ └── ci-build-tests.yml ├── .gitignore ├── .gitmodules ├── CONTRIBUTING.md ├── COPYING ├── MANIFEST.in ├── README.md ├── cli.py ├── clustering-vt └── clustering-vt.cpp ├── contrib ├── geojson │ └── 0.4.3 │ │ └── include │ │ └── mapbox │ │ ├── geojson.hpp │ │ ├── geojson │ │ └── rapidjson.hpp │ │ └── geojson_impl.hpp ├── geometry │ └── 1.0.0 │ │ └── include │ │ └── mapbox │ │ ├── feature.hpp │ │ ├── geometry.hpp │ │ ├── geometry │ │ ├── box.hpp │ │ ├── empty.hpp │ │ ├── envelope.hpp │ │ ├── for_each_point.hpp │ │ ├── geometry.hpp │ │ ├── line_string.hpp │ │ ├── multi_line_string.hpp │ │ ├── multi_point.hpp │ │ ├── multi_polygon.hpp │ │ ├── point.hpp │ │ ├── point_arithmetic.hpp │ │ └── polygon.hpp │ │ └── geometry_io.hpp ├── kdbush │ └── 0.1.3 │ │ ├── LICENSE │ │ ├── README.md │ │ └── include │ │ └── kdbush.hpp ├── protozero │ └── 1.7.0 │ │ ├── LICENSE.md │ │ └── include │ │ └── protozero │ │ ├── basic_pbf_builder.hpp │ │ ├── basic_pbf_writer.hpp │ │ ├── buffer_fixed.hpp │ │ ├── buffer_string.hpp │ │ ├── buffer_tmpl.hpp │ │ ├── buffer_vector.hpp │ │ ├── byteswap.hpp │ │ ├── config.hpp │ │ ├── data_view.hpp │ │ ├── exception.hpp │ │ ├── iterators.hpp │ │ ├── pbf_builder.hpp │ │ ├── pbf_message.hpp │ │ ├── pbf_reader.hpp │ │ ├── pbf_writer.hpp │ │ ├── types.hpp │ │ ├── varint.hpp │ │ └── version.hpp ├── rapidjson │ ├── 1.1.0 │ │ └── include │ │ │ └── rapidjson │ │ │ ├── allocators.h │ │ │ ├── document.h │ │ │ ├── encodedstream.h │ │ │ ├── encodings.h │ │ │ ├── error │ │ │ ├── en.h │ │ │ └── error.h │ │ │ ├── filereadstream.h │ │ │ ├── filewritestream.h │ │ │ ├── fwd.h │ │ │ ├── internal │ │ │ ├── biginteger.h │ │ │ ├── diyfp.h │ │ │ ├── dtoa.h │ │ │ ├── ieee754.h │ │ │ ├── itoa.h │ │ │ ├── meta.h │ │ │ ├── pow10.h │ │ │ ├── regex.h │ │ │ ├── stack.h │ │ │ ├── strfunc.h │ │ │ ├── strtod.h │ │ │ └── swap.h │ │ │ ├── istreamwrapper.h │ │ │ ├── memorybuffer.h │ │ │ ├── memorystream.h │ │ │ ├── msinttypes │ │ │ ├── inttypes.h │ │ │ └── stdint.h │ │ │ ├── ostreamwrapper.h │ │ │ ├── pointer.h │ │ │ ├── prettywriter.h │ │ │ ├── rapidjson.h │ │ │ ├── reader.h │ │ │ ├── schema.h │ │ │ ├── stream.h │ │ │ ├── stringbuffer.h │ │ │ └── writer.h │ └── LICENSE ├── supercluster │ └── 0.3.2 │ │ ├── LICENSE │ │ ├── README.md │ │ └── include │ │ └── supercluster.hpp ├── variant │ └── 1.2.0 │ │ ├── LICENSE │ │ ├── README.md │ │ └── include │ │ └── mapbox │ │ ├── optional.hpp │ │ ├── recursive_wrapper.hpp │ │ ├── variant.hpp │ │ ├── variant_cast.hpp │ │ ├── variant_io.hpp │ │ └── variant_visitor.hpp └── vtzero │ └── 1.1.0 │ ├── LICENSE │ └── include │ └── vtzero │ ├── builder.hpp │ ├── builder_impl.hpp │ ├── encoded_property_value.hpp │ ├── exception.hpp │ ├── feature.hpp │ ├── feature_builder_impl.hpp │ ├── geometry.hpp │ ├── index.hpp │ ├── layer.hpp │ ├── output.hpp │ ├── property.hpp │ ├── property_mapper.hpp │ ├── property_value.hpp │ ├── types.hpp │ ├── vector_tile.hpp │ └── version.hpp ├── docs ├── Add-New-Rule.md ├── Dynamic-Value.md ├── Overview.md ├── Pipes.md └── YAML-Rule-Specification.md ├── pyproject.toml ├── setup.py ├── src └── nominatim_data_analyser │ ├── __init__.py │ ├── cli.py │ ├── config.py │ ├── core │ ├── __init__.py │ ├── assembler │ │ ├── __init__.py │ │ ├── pipe_factory.py │ │ └── pipeline_assembler.py │ ├── core.py │ ├── deconstructor │ │ ├── __init__.py │ │ └── pipeline_deconstructor.py │ ├── dynamic_value │ │ ├── __init__.py │ │ ├── dynamic_value.py │ │ ├── resolver.py │ │ ├── switch.py │ │ └── variable.py │ ├── exceptions │ │ ├── __init__.py │ │ └── yaml_syntax_exception.py │ ├── model │ │ ├── __init__.py │ │ ├── geometry.py │ │ └── node.py │ ├── pipe.py │ ├── pipes │ │ ├── __init__.py │ │ ├── data_fetching │ │ │ ├── __init__.py │ │ │ └── sql_processor.py │ │ ├── data_processing │ │ │ ├── __init__.py │ │ │ ├── geometry_converter.py │ │ │ └── loop_data_processor.py │ │ ├── filling_pipe.py │ │ ├── output_formatters │ │ │ ├── __init__.py │ │ │ ├── clusters_vt_formatter.py │ │ │ ├── geojson_feature_converter.py │ │ │ ├── geojson_formatter.py │ │ │ ├── osmoscope_layer_formatter.py │ │ │ └── vector_tile_formatter.py │ │ └── rules_specific_pipes │ │ │ ├── __init__.py │ │ │ ├── addr_house_number_no_digit │ │ │ └── digits_filter.py │ │ │ ├── duplicate_label_role │ │ │ └── custom_feature_converter.py │ │ │ ├── place_nodes_close │ │ │ └── custom_feature_converter.py │ │ │ └── same_wikidata │ │ │ └── custom_feature_converter.py │ ├── qa_rule │ │ ├── __init__.py │ │ └── execution_context.py │ └── yaml_logic │ │ ├── __init__.py │ │ └── yaml_loader.py │ ├── default_config.yaml │ ├── logger │ ├── logger.py │ └── timer.py │ ├── py.typed │ └── rules_specifications │ ├── BA_way_not_part_relation.yaml │ ├── addr_housenumber_no_digit.yaml │ ├── addr_place_and_street.yaml │ ├── addr_place_or_street_rank28.yaml │ ├── addr_street_wrong_name.yaml │ ├── bad_interpolations.yaml │ ├── duplicate_label_role.yaml │ ├── no_admin_level.yaml │ ├── place_nodes_close.yaml │ ├── postcode_bad_format.yaml │ └── same_wikidata.yaml └── tests ├── __init__.py ├── assembler ├── test_pipe_factory.py └── test_pipeline_assembler.py ├── config └── test_config.py ├── conftest.py ├── core ├── rules │ ├── rule1.yaml │ └── rule2.yaml └── test_core.py ├── deconstructor └── test_pipeline_deconstructor.py ├── dynamic_value ├── test_resolver.py ├── test_switch.py └── test_variable.py ├── model └── test_node.py ├── pipes ├── output_formatters │ ├── test_geojson_feature_converter.py │ ├── test_geojson_formatter.py │ ├── test_osmoscope_layer_formatter.py │ └── test_vector_tile_formatter.py ├── rules_specific_pipes │ ├── addr_house_number_no_digit │ │ └── test_digits_filter.py │ ├── duplicate_label_role │ │ └── test_duplicate_label_role_CFC.py │ ├── place_nodes_close │ │ └── test_place_nodes_close_CFC.py │ └── same_wikidata │ │ └── test_same_wikidata_CFC.py ├── test_geometry_converter.py ├── test_loop_data_processor.py ├── test_pipe.py └── test_sql_processor.py └── yaml_logic ├── test_yaml_loader.py └── yaml ├── test_construct_sub_pipeline.yaml ├── test_construct_switch.yaml ├── test_construct_variable.yaml ├── test_load_wrong_yaml.yaml └── test_load_yaml.yaml /.coveragerc: -------------------------------------------------------------------------------- 1 | [report] 2 | exclude_lines = 3 | pragma: no cover 4 | @abstract -------------------------------------------------------------------------------- /.github/workflows/ci-build-tests.yml: -------------------------------------------------------------------------------- 1 | name: CI Build-Tests 2 | on: [push, pull_request] 3 | jobs: 4 | build-tests: 5 | runs-on: ubuntu-latest 6 | steps: 7 | - uses: actions/checkout@v4 8 | - name: Set up Python 3.10 9 | uses: actions/setup-python@v5 10 | with: 11 | python-version: "3.10" 12 | - name: Install python dependencies 13 | run: | 14 | python -m pip install --upgrade pip 15 | pip install flake8 pytest pybind11 mypy types-psycopg2 types-pyyaml 16 | pip install . 17 | - name: Lint with flake8 18 | run: | 19 | # stop the build if there are Python syntax errors or undefined names 20 | flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics 21 | # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide 22 | flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --exclude tests,build,__init__.py --extend-ignore=E302,E722 --statistics 23 | - name: Check typing 24 | run: mypy 25 | - name: Install PostgreSQL 26 | run: | 27 | sudo apt-get update 28 | sudo apt-get install postgresql postgresql-client 29 | sudo systemctl restart postgresql 30 | sudo -u postgres createuser -s runner 31 | - name: Test with pytest 32 | run: | 33 | python3 setup.py build 34 | pytest tests 35 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | src/nominatim_data_analyser.egg-info 3 | dist 4 | 5 | src/nominatim_data_analyser/config/config.yaml 6 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/osm-search/Nominatim-Data-Analyser/57cd6060ab506e3f7bf0b0e50f1fd1aa25e91567/.gitmodules -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Reporting errors in the rules 2 | 3 | We do not have a system to automatically report data as false positive yet. 4 | 5 | If you find a big chunk of data for a rule which are not real errors and that it can possibly 6 | be fix by modifing a bit the rule's logic, please open an [Issue](https://github.com/AntoJvlt/Nominatim-Data-Analyser/issues) 7 | to discuss it. 8 | 9 | # Contributing to this repository 10 | 11 | Before you start: 12 | 13 | * Follow the [Installation procedure](https://github.com/AntoJvlt/Nominatim-Data-Analyser#installation-procedure). 14 | * Check the [Documentation](docs/Overview.md). 15 | 16 | If you want to add a new rule, follow the [Add a new rule](docs/Add-New-Rule.md) chapter. -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include src/nominatim_data_analyser/rules_specifications/*.yaml 2 | include src/nominatim_data_analyser/default_config.yaml 3 | include src/nominatim_data_analyser/py.typed 4 | recursive-include contrib * 5 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Nominatim-Data-Analyser ![CI Build-Tests](https://github.com/osm-search/Nominatim-Data-Analyser/actions/workflows/ci-build-tests.yml/badge.svg) 2 | 3 | The Nominatim Data Analyser is a QA tool used to scan the nominatim database and extract 4 | suspect data from it. These data are then presented to mappers through a [visual interface](https://nominatim.org/qa/) so that they can correct them directly. 5 | 6 | # Frontend 7 | 8 | The repository containing the frontend of the tool can be found [there](https://github.com/osm-search/Nominatim-Data-Analyser-Frontend). 9 | 10 | # Installation procedure 11 | 12 | Clone this repository by running: 13 | 14 | ``` 15 | git clone https://github.com/osm-search/Nominatim-Data-Analyser 16 | ``` 17 | 18 | Then you can compile everything with 19 | 20 | python3 setup.py build 21 | 22 | Or you can directly compile and install the analyser with 23 | 24 | pip install . 25 | 26 | ## Database 27 | 28 | Make sure to have a Nominatim database set up on your server. You can change 29 | the database DSN by supplying a custom config.yaml file. 30 | 31 | ## Configuration file 32 | 33 | Some parts of the analyser are configurable. You can find the default 34 | configuration in `src/nominatim_data_analyser/default_config.yaml`. To 35 | modify the configuration create a copy of the file with the modified 36 | values and put it as `config.yaml` in the directory where the analyser 37 | is executed. 38 | 39 | ## Frontend set up 40 | 41 | To set up the frontend, please check the frontend [repository](https://github.com/osm-search/Nominatim-Data-Analyser-Frontend). 42 | 43 | For the webapp to fetch the data extracted by the analyser, you need to serve the `````` defined in ```analyser/config/config.yaml``` of the QA Data Analyser Tool with a web server. It should be accessible through the `````` also defined in the configuration of the QA Data Analyser Tool. 44 | 45 | # Running the analyser 46 | 47 | Analysis is run with the nominatim-data-analyser tool: 48 | 49 | * --execute-all: Executes all QA rules. 50 | * --filter [rules_names…]: Filters some QA rules so they are not executed. 51 | * --execute-one : Executes the given QA rule. 52 | * --config: Set a custom location for the configuration file. 53 | 54 | During development you can run the same tool directly from the source tree 55 | after having built everything using the supplied `cli.py`. 56 | 57 | # Tests 58 | 59 | [Pytest](https://docs.pytest.org/en/6.2.x/getting-started.html) is used for the tests and it should be installed: 60 | 61 | ``` 62 | pip install pytest 63 | ``` 64 | 65 | To run the tests for the analyser: execute ```pytest``` command at the root folder of the project. 66 | 67 | # Reporting errors in the rule 68 | 69 | To report some errors in the rule, please follow the [CONTRIBUTING.md](CONTRIBUTING.md) 70 | 71 | # Documentation 72 | 73 | An advanced documentation of the tool can be found in the [Overview.md](docs/Overview.md) file. 74 | -------------------------------------------------------------------------------- /cli.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | import sys 3 | import sysconfig 4 | from pathlib import Path 5 | 6 | SRC_DIR = Path(__file__, '..').resolve() 7 | 8 | BUILD_DIR = f"build/lib.{sysconfig.get_platform()}-{sys.version_info[0]}.{sys.version_info[1]}" 9 | 10 | if not (SRC_DIR / BUILD_DIR).exists(): 11 | BUILD_DIR = f"build/lib.{sysconfig.get_platform()}-{sys.implementation.cache_tag}" 12 | 13 | if (SRC_DIR / BUILD_DIR).exists(): 14 | sys.path.insert(0, str(SRC_DIR / BUILD_DIR)) 15 | 16 | 17 | from nominatim_data_analyser.cli import cli # noqa 18 | 19 | sys.exit(cli()) 20 | -------------------------------------------------------------------------------- /contrib/geojson/0.4.3/include/mapbox/geojson.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | namespace mapbox { 8 | namespace geojson { 9 | 10 | using empty = mapbox::geometry::empty; 11 | using point = mapbox::geometry::point; 12 | using multi_point = mapbox::geometry::multi_point; 13 | using line_string = mapbox::geometry::line_string; 14 | using linear_ring = mapbox::geometry::linear_ring; 15 | using multi_line_string = mapbox::geometry::multi_line_string; 16 | using polygon = mapbox::geometry::polygon; 17 | using multi_polygon = mapbox::geometry::multi_polygon; 18 | using geometry = mapbox::geometry::geometry; 19 | using geometry_collection = mapbox::geometry::geometry_collection; 20 | 21 | using value = mapbox::feature::value; 22 | using null_value_t = mapbox::feature::null_value_t; 23 | using identifier = mapbox::feature::identifier; 24 | using feature = mapbox::feature::feature; 25 | using feature_collection = mapbox::feature::feature_collection; 26 | 27 | // Parse inputs of known types. Instantiations are provided for geometry, feature, and 28 | // feature_collection. 29 | template 30 | T parse(const std::string &); 31 | 32 | // Parse any GeoJSON type. 33 | using geojson = mapbox::util::variant; 34 | geojson parse(const std::string &); 35 | 36 | // Stringify inputs of known types. Instantiations are provided for geometry, feature, and 37 | // feature_collection. 38 | template 39 | std::string stringify(const T &); 40 | 41 | // Stringify any GeoJSON type. 42 | std::string stringify(const geojson &); 43 | 44 | } // namespace geojson 45 | } // namespace mapbox 46 | -------------------------------------------------------------------------------- /contrib/geojson/0.4.3/include/mapbox/geojson/rapidjson.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | namespace mapbox { 7 | namespace geojson { 8 | 9 | // Use the CrtAllocator, because the MemoryPoolAllocator is broken on ARM 10 | // https://github.com/miloyip/rapidjson/issues/200, 301, 388 11 | using rapidjson_allocator = rapidjson::CrtAllocator; 12 | using rapidjson_document = rapidjson::GenericDocument, rapidjson_allocator>; 13 | using rapidjson_value = rapidjson::GenericValue, rapidjson_allocator>; 14 | 15 | // Convert inputs of known types. Instantiations are provided for geometry, feature, and 16 | // feature_collection. 17 | template 18 | T convert(const rapidjson_value &); 19 | 20 | // Convert any GeoJSON type. 21 | geojson convert(const rapidjson_value &); 22 | 23 | // Convert back to rapidjson value. Instantiations are provided for geometry, feature, and 24 | // feature_collection. 25 | template 26 | rapidjson_value convert(const T &, rapidjson_allocator&); 27 | 28 | // Convert any GeoJSON type. 29 | rapidjson_value convert(const geojson &, rapidjson_allocator&); 30 | 31 | } // namespace geojson 32 | } // namespace mapbox 33 | -------------------------------------------------------------------------------- /contrib/geometry/1.0.0/include/mapbox/geometry.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | -------------------------------------------------------------------------------- /contrib/geometry/1.0.0/include/mapbox/geometry/box.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | namespace mapbox { 6 | namespace geometry { 7 | 8 | template 9 | struct box 10 | { 11 | using coordinate_type = T; 12 | using point_type = point; 13 | 14 | constexpr box(point_type const& min_, point_type const& max_) 15 | : min(min_), max(max_) 16 | { 17 | } 18 | 19 | point_type min; 20 | point_type max; 21 | }; 22 | 23 | template 24 | constexpr bool operator==(box const& lhs, box const& rhs) 25 | { 26 | return lhs.min == rhs.min && lhs.max == rhs.max; 27 | } 28 | 29 | template 30 | constexpr bool operator!=(box const& lhs, box const& rhs) 31 | { 32 | return lhs.min != rhs.min || lhs.max != rhs.max; 33 | } 34 | 35 | } // namespace geometry 36 | } // namespace mapbox 37 | -------------------------------------------------------------------------------- /contrib/geometry/1.0.0/include/mapbox/geometry/empty.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | namespace mapbox { 4 | namespace geometry { 5 | 6 | struct empty 7 | { 8 | }; // this Geometry type represents the empty point set, ∅, for the coordinate space (OGC Simple Features). 9 | 10 | constexpr bool operator==(empty, empty) { return true; } 11 | constexpr bool operator!=(empty, empty) { return false; } 12 | constexpr bool operator<(empty, empty) { return false; } 13 | constexpr bool operator>(empty, empty) { return false; } 14 | constexpr bool operator<=(empty, empty) { return true; } 15 | constexpr bool operator>=(empty, empty) { return true; } 16 | 17 | } // namespace geometry 18 | } // namespace mapbox 19 | -------------------------------------------------------------------------------- /contrib/geometry/1.0.0/include/mapbox/geometry/envelope.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | #include 7 | 8 | namespace mapbox { 9 | namespace geometry { 10 | 11 | template 12 | box envelope(G const& geometry) 13 | { 14 | using limits = std::numeric_limits; 15 | 16 | T min_t = limits::has_infinity ? -limits::infinity() : limits::min(); 17 | T max_t = limits::has_infinity ? limits::infinity() : limits::max(); 18 | 19 | point min(max_t, max_t); 20 | point max(min_t, min_t); 21 | 22 | for_each_point(geometry, [&](point const& point) { 23 | if (min.x > point.x) min.x = point.x; 24 | if (min.y > point.y) min.y = point.y; 25 | if (max.x < point.x) max.x = point.x; 26 | if (max.y < point.y) max.y = point.y; 27 | }); 28 | 29 | return box(min, max); 30 | } 31 | 32 | } // namespace geometry 33 | } // namespace mapbox 34 | -------------------------------------------------------------------------------- /contrib/geometry/1.0.0/include/mapbox/geometry/for_each_point.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | namespace mapbox { 6 | namespace geometry { 7 | 8 | template 9 | void for_each_point(mapbox::geometry::empty const&, F&&) 10 | { 11 | } 12 | 13 | template 14 | auto for_each_point(Point&& point, F&& f) 15 | -> decltype(point.x, point.y, void()) 16 | { 17 | f(std::forward(point)); 18 | } 19 | 20 | template 21 | auto for_each_point(Container&& container, F&& f) 22 | -> decltype(container.begin(), container.end(), void()); 23 | 24 | template 25 | void for_each_point(mapbox::util::variant const& geom, F&& f) 26 | { 27 | mapbox::util::variant::visit(geom, [&](auto const& g) { 28 | for_each_point(g, f); 29 | }); 30 | } 31 | 32 | template 33 | void for_each_point(mapbox::util::variant& geom, F&& f) 34 | { 35 | mapbox::util::variant::visit(geom, [&](auto& g) { 36 | for_each_point(g, f); 37 | }); 38 | } 39 | 40 | template 41 | auto for_each_point(Container&& container, F&& f) 42 | -> decltype(container.begin(), container.end(), void()) 43 | { 44 | for (auto& e : container) 45 | { 46 | for_each_point(e, f); 47 | } 48 | } 49 | 50 | } // namespace geometry 51 | } // namespace mapbox 52 | -------------------------------------------------------------------------------- /contrib/geometry/1.0.0/include/mapbox/geometry/geometry.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include 12 | 13 | // stl 14 | #include 15 | 16 | namespace mapbox { 17 | namespace geometry { 18 | 19 | template class Cont = std::vector> 20 | struct geometry_collection; 21 | 22 | template class Cont = std::vector> 23 | using geometry_base = mapbox::util::variant, 25 | line_string, 26 | polygon, 27 | multi_point, 28 | multi_line_string, 29 | multi_polygon, 30 | geometry_collection>; 31 | 32 | template class Cont = std::vector> 33 | struct geometry : geometry_base 34 | { 35 | using coordinate_type = T; 36 | using geometry_base::geometry_base; 37 | }; 38 | 39 | template class Cont> 40 | struct geometry_collection : Cont> 41 | { 42 | using coordinate_type = T; 43 | using geometry_type = geometry; 44 | using container_type = Cont; 45 | using size_type = typename container_type::size_type; 46 | 47 | template 48 | geometry_collection(Args&&... args) : container_type(std::forward(args)...) 49 | { 50 | } 51 | geometry_collection(std::initializer_list args) 52 | : container_type(std::move(args)) {} 53 | }; 54 | 55 | } // namespace geometry 56 | } // namespace mapbox 57 | -------------------------------------------------------------------------------- /contrib/geometry/1.0.0/include/mapbox/geometry/line_string.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | // mapbox 4 | #include 5 | // stl 6 | #include 7 | 8 | namespace mapbox { 9 | namespace geometry { 10 | 11 | template class Cont = std::vector> 12 | struct line_string : Cont> 13 | { 14 | using coordinate_type = T; 15 | using point_type = point; 16 | using container_type = Cont; 17 | using size_type = typename container_type::size_type; 18 | 19 | template 20 | line_string(Args&&... args) : container_type(std::forward(args)...) 21 | { 22 | } 23 | line_string(std::initializer_list args) 24 | : container_type(std::move(args)) {} 25 | }; 26 | 27 | } // namespace geometry 28 | } // namespace mapbox 29 | -------------------------------------------------------------------------------- /contrib/geometry/1.0.0/include/mapbox/geometry/multi_line_string.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | // mapbox 4 | #include 5 | // stl 6 | #include 7 | 8 | namespace mapbox { 9 | namespace geometry { 10 | 11 | template class Cont = std::vector> 12 | struct multi_line_string : Cont> 13 | { 14 | using coordinate_type = T; 15 | using line_string_type = line_string; 16 | using container_type = Cont; 17 | using size_type = typename container_type::size_type; 18 | 19 | template 20 | multi_line_string(Args&&... args) : container_type(std::forward(args)...) 21 | { 22 | } 23 | multi_line_string(std::initializer_list args) 24 | : container_type(std::move(args)) {} 25 | }; 26 | 27 | } // namespace geometry 28 | } // namespace mapbox 29 | -------------------------------------------------------------------------------- /contrib/geometry/1.0.0/include/mapbox/geometry/multi_point.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | // mapbox 4 | #include 5 | // stl 6 | #include 7 | 8 | namespace mapbox { 9 | namespace geometry { 10 | 11 | template class Cont = std::vector> 12 | struct multi_point : Cont> 13 | { 14 | using coordinate_type = T; 15 | using point_type = point; 16 | using container_type = Cont; 17 | using size_type = typename container_type::size_type; 18 | 19 | template 20 | multi_point(Args&&... args) : container_type(std::forward(args)...) 21 | { 22 | } 23 | multi_point(std::initializer_list args) 24 | : container_type(std::move(args)) {} 25 | }; 26 | 27 | } // namespace geometry 28 | } // namespace mapbox 29 | -------------------------------------------------------------------------------- /contrib/geometry/1.0.0/include/mapbox/geometry/multi_polygon.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | // mapbox 4 | #include 5 | // stl 6 | #include 7 | 8 | namespace mapbox { 9 | namespace geometry { 10 | 11 | template class Cont = std::vector> 12 | struct multi_polygon : Cont> 13 | { 14 | using coordinate_type = T; 15 | using polygon_type = polygon; 16 | using container_type = Cont; 17 | using size_type = typename container_type::size_type; 18 | 19 | template 20 | multi_polygon(Args&&... args) : container_type(std::forward(args)...) 21 | { 22 | } 23 | multi_polygon(std::initializer_list args) 24 | : container_type(std::move(args)) {} 25 | }; 26 | 27 | } // namespace geometry 28 | } // namespace mapbox 29 | -------------------------------------------------------------------------------- /contrib/geometry/1.0.0/include/mapbox/geometry/point.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | namespace mapbox { 4 | namespace geometry { 5 | 6 | template 7 | struct point 8 | { 9 | using coordinate_type = T; 10 | 11 | constexpr point() 12 | : x(), y() 13 | { 14 | } 15 | constexpr point(T x_, T y_) 16 | : x(x_), y(y_) 17 | { 18 | } 19 | 20 | T x; 21 | T y; 22 | }; 23 | 24 | #pragma GCC diagnostic push 25 | #pragma GCC diagnostic ignored "-Wfloat-equal" 26 | 27 | template 28 | constexpr bool operator==(point const& lhs, point const& rhs) 29 | { 30 | return lhs.x == rhs.x && lhs.y == rhs.y; 31 | } 32 | 33 | #pragma GCC diagnostic pop 34 | 35 | template 36 | constexpr bool operator!=(point const& lhs, point const& rhs) 37 | { 38 | return !(lhs == rhs); 39 | } 40 | 41 | } // namespace geometry 42 | } // namespace mapbox 43 | -------------------------------------------------------------------------------- /contrib/geometry/1.0.0/include/mapbox/geometry/point_arithmetic.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | namespace mapbox { 4 | namespace geometry { 5 | 6 | template 7 | point operator+(point const& lhs, point const& rhs) 8 | { 9 | return point(lhs.x + rhs.x, lhs.y + rhs.y); 10 | } 11 | 12 | template 13 | point operator+(point const& lhs, T const& rhs) 14 | { 15 | return point(lhs.x + rhs, lhs.y + rhs); 16 | } 17 | 18 | template 19 | point operator-(point const& lhs, point const& rhs) 20 | { 21 | return point(lhs.x - rhs.x, lhs.y - rhs.y); 22 | } 23 | 24 | template 25 | point operator-(point const& lhs, T const& rhs) 26 | { 27 | return point(lhs.x - rhs, lhs.y - rhs); 28 | } 29 | 30 | template 31 | point operator*(point const& lhs, point const& rhs) 32 | { 33 | return point(lhs.x * rhs.x, lhs.y * rhs.y); 34 | } 35 | 36 | template 37 | point operator*(point const& lhs, T const& rhs) 38 | { 39 | return point(lhs.x * rhs, lhs.y * rhs); 40 | } 41 | 42 | template 43 | point operator/(point const& lhs, point const& rhs) 44 | { 45 | return point(lhs.x / rhs.x, lhs.y / rhs.y); 46 | } 47 | 48 | template 49 | point operator/(point const& lhs, T const& rhs) 50 | { 51 | return point(lhs.x / rhs, lhs.y / rhs); 52 | } 53 | 54 | template 55 | point& operator+=(point& lhs, point const& rhs) 56 | { 57 | lhs.x += rhs.x; 58 | lhs.y += rhs.y; 59 | return lhs; 60 | } 61 | 62 | template 63 | point& operator+=(point& lhs, T const& rhs) 64 | { 65 | lhs.x += rhs; 66 | lhs.y += rhs; 67 | return lhs; 68 | } 69 | 70 | template 71 | point& operator-=(point& lhs, point const& rhs) 72 | { 73 | lhs.x -= rhs.x; 74 | lhs.y -= rhs.y; 75 | return lhs; 76 | } 77 | 78 | template 79 | point& operator-=(point& lhs, T const& rhs) 80 | { 81 | lhs.x -= rhs; 82 | lhs.y -= rhs; 83 | return lhs; 84 | } 85 | 86 | template 87 | point& operator*=(point& lhs, point const& rhs) 88 | { 89 | lhs.x *= rhs.x; 90 | lhs.y *= rhs.y; 91 | return lhs; 92 | } 93 | 94 | template 95 | point& operator*=(point& lhs, T const& rhs) 96 | { 97 | lhs.x *= rhs; 98 | lhs.y *= rhs; 99 | return lhs; 100 | } 101 | 102 | template 103 | point& operator/=(point& lhs, point const& rhs) 104 | { 105 | lhs.x /= rhs.x; 106 | lhs.y /= rhs.y; 107 | return lhs; 108 | } 109 | 110 | template 111 | point& operator/=(point& lhs, T const& rhs) 112 | { 113 | lhs.x /= rhs; 114 | lhs.y /= rhs; 115 | return lhs; 116 | } 117 | 118 | } // namespace geometry 119 | } // namespace mapbox 120 | -------------------------------------------------------------------------------- /contrib/geometry/1.0.0/include/mapbox/geometry/polygon.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | // mapbox 4 | #include 5 | 6 | // stl 7 | #include 8 | 9 | namespace mapbox { 10 | namespace geometry { 11 | 12 | template class Cont = std::vector> 13 | struct linear_ring : Cont> 14 | { 15 | using coordinate_type = T; 16 | using point_type = point; 17 | using container_type = Cont; 18 | using size_type = typename container_type::size_type; 19 | 20 | template 21 | linear_ring(Args&&... args) : container_type(std::forward(args)...) 22 | { 23 | } 24 | linear_ring(std::initializer_list args) 25 | : container_type(std::move(args)) {} 26 | }; 27 | 28 | template class Cont = std::vector> 29 | struct polygon : Cont> 30 | { 31 | using coordinate_type = T; 32 | using linear_ring_type = linear_ring; 33 | using container_type = Cont; 34 | using size_type = typename container_type::size_type; 35 | 36 | template 37 | polygon(Args&&... args) : container_type(std::forward(args)...) 38 | { 39 | } 40 | polygon(std::initializer_list args) 41 | : container_type(std::move(args)) {} 42 | }; 43 | 44 | } // namespace geometry 45 | } // namespace mapbox 46 | -------------------------------------------------------------------------------- /contrib/geometry/1.0.0/include/mapbox/geometry_io.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | #include 7 | #include 8 | 9 | namespace mapbox { 10 | namespace geometry { 11 | 12 | std::ostream& operator<<(std::ostream& os, const empty&) 13 | { 14 | return os << "[]"; 15 | } 16 | 17 | template 18 | std::ostream& operator<<(std::ostream& os, const point& point) 19 | { 20 | return os << "[" << point.x << "," << point.y << "]"; 21 | } 22 | 23 | template class C, class... Args> 24 | std::ostream& operator<<(std::ostream& os, const C& cont) 25 | { 26 | os << "["; 27 | for (auto it = cont.cbegin();;) 28 | { 29 | os << *it; 30 | if (++it == cont.cend()) 31 | { 32 | break; 33 | } 34 | os << ","; 35 | } 36 | return os << "]"; 37 | } 38 | 39 | template 40 | std::ostream& operator<<(std::ostream& os, const line_string& geom) 41 | { 42 | return os << static_cast::container_type>(geom); 43 | } 44 | 45 | template 46 | std::ostream& operator<<(std::ostream& os, const linear_ring& geom) 47 | { 48 | return os << static_cast::container_type>(geom); 49 | } 50 | 51 | template 52 | std::ostream& operator<<(std::ostream& os, const polygon& geom) 53 | { 54 | return os << static_cast::container_type>(geom); 55 | } 56 | 57 | template 58 | std::ostream& operator<<(std::ostream& os, const multi_point& geom) 59 | { 60 | return os << static_cast::container_type>(geom); 61 | } 62 | 63 | template 64 | std::ostream& operator<<(std::ostream& os, const multi_line_string& geom) 65 | { 66 | return os << static_cast::container_type>(geom); 67 | } 68 | 69 | template 70 | std::ostream& operator<<(std::ostream& os, const multi_polygon& geom) 71 | { 72 | return os << static_cast::container_type>(geom); 73 | } 74 | 75 | template 76 | std::ostream& operator<<(std::ostream& os, const geometry& geom) 77 | { 78 | geometry::visit(geom, [&](const auto& g) { os << g; }); 79 | return os; 80 | } 81 | 82 | template 83 | std::ostream& operator<<(std::ostream& os, const geometry_collection& geom) 84 | { 85 | return os << static_cast::container_type>(geom); 86 | } 87 | 88 | } // namespace geometry 89 | 90 | namespace feature { 91 | 92 | std::ostream& operator<<(std::ostream& os, const null_value_t&) 93 | { 94 | return os << "[]"; 95 | } 96 | 97 | } // namespace feature 98 | } // namespace mapbox 99 | -------------------------------------------------------------------------------- /contrib/kdbush/0.1.3/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016, Vladimir Agafonkin 2 | 3 | Permission to use, copy, modify, and/or distribute this software for any 4 | purpose with or without fee is hereby granted, provided that the above 5 | copyright notice and this permission notice appear in all copies. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 8 | WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 9 | MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 10 | ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 11 | WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 12 | ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 13 | OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 14 | -------------------------------------------------------------------------------- /contrib/kdbush/0.1.3/README.md: -------------------------------------------------------------------------------- 1 | A C++11 port of [kdbush](https://github.com/mourner/kdbush), a fast static spatial index for 2D points. 2 | 3 | [![Build Status](https://travis-ci.org/mourner/kdbush.hpp.svg?branch=master)](https://travis-ci.org/mourner/kdbush.hpp) 4 | -------------------------------------------------------------------------------- /contrib/protozero/1.7.0/LICENSE.md: -------------------------------------------------------------------------------- 1 | protozero copyright (c) Mapbox. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are 5 | met: 6 | 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above copyright 10 | notice, this list of conditions and the following disclaimer in 11 | the documentation and/or other materials provided with the 12 | distribution. 13 | 14 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS 15 | IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 16 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 17 | PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 18 | CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 19 | EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 20 | PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 21 | PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 22 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 23 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 24 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 | -------------------------------------------------------------------------------- /contrib/protozero/1.7.0/include/protozero/buffer_string.hpp: -------------------------------------------------------------------------------- 1 | #ifndef PROTOZERO_BUFFER_STRING_HPP 2 | #define PROTOZERO_BUFFER_STRING_HPP 3 | 4 | /***************************************************************************** 5 | 6 | protozero - Minimalistic protocol buffer decoder and encoder in C++. 7 | 8 | This file is from https://github.com/mapbox/protozero where you can find more 9 | documentation. 10 | 11 | *****************************************************************************/ 12 | 13 | /** 14 | * @file buffer_string.hpp 15 | * 16 | * @brief Contains the customization points for buffer implementation based 17 | * on std::string 18 | */ 19 | 20 | #include "buffer_tmpl.hpp" 21 | 22 | #include 23 | #include 24 | #include 25 | 26 | namespace protozero { 27 | 28 | // Implementation of buffer customizations points for std::string 29 | 30 | /// @cond INTERNAL 31 | template <> 32 | struct buffer_customization { 33 | 34 | static std::size_t size(const std::string* buffer) noexcept { 35 | return buffer->size(); 36 | } 37 | 38 | static void append(std::string* buffer, const char* data, std::size_t count) { 39 | buffer->append(data, count); 40 | } 41 | 42 | static void append_zeros(std::string* buffer, std::size_t count) { 43 | buffer->append(count, '\0'); 44 | } 45 | 46 | static void resize(std::string* buffer, std::size_t size) { 47 | protozero_assert(size < buffer->size()); 48 | buffer->resize(size); 49 | } 50 | 51 | static void reserve_additional(std::string* buffer, std::size_t size) { 52 | buffer->reserve(buffer->size() + size); 53 | } 54 | 55 | static void erase_range(std::string* buffer, std::size_t from, std::size_t to) { 56 | protozero_assert(from <= buffer->size()); 57 | protozero_assert(to <= buffer->size()); 58 | protozero_assert(from <= to); 59 | buffer->erase(std::next(buffer->begin(), from), std::next(buffer->begin(), to)); 60 | } 61 | 62 | static char* at_pos(std::string* buffer, std::size_t pos) { 63 | protozero_assert(pos <= buffer->size()); 64 | return (&*buffer->begin()) + pos; 65 | } 66 | 67 | static void push_back(std::string* buffer, char ch) { 68 | buffer->push_back(ch); 69 | } 70 | 71 | }; 72 | /// @endcond 73 | 74 | } // namespace protozero 75 | 76 | #endif // PROTOZERO_BUFFER_STRING_HPP 77 | -------------------------------------------------------------------------------- /contrib/protozero/1.7.0/include/protozero/buffer_vector.hpp: -------------------------------------------------------------------------------- 1 | #ifndef PROTOZERO_BUFFER_VECTOR_HPP 2 | #define PROTOZERO_BUFFER_VECTOR_HPP 3 | 4 | /***************************************************************************** 5 | 6 | protozero - Minimalistic protocol buffer decoder and encoder in C++. 7 | 8 | This file is from https://github.com/mapbox/protozero where you can find more 9 | documentation. 10 | 11 | *****************************************************************************/ 12 | 13 | /** 14 | * @file buffer_vector.hpp 15 | * 16 | * @brief Contains the customization points for buffer implementation based 17 | * on std::vector 18 | */ 19 | 20 | #include "buffer_tmpl.hpp" 21 | 22 | #include 23 | #include 24 | #include 25 | 26 | namespace protozero { 27 | 28 | // Implementation of buffer customizations points for std::vector 29 | 30 | /// @cond INTERNAL 31 | template <> 32 | struct buffer_customization> { 33 | 34 | static std::size_t size(const std::vector* buffer) noexcept { 35 | return buffer->size(); 36 | } 37 | 38 | static void append(std::vector* buffer, const char* data, std::size_t count) { 39 | buffer->insert(buffer->end(), data, data + count); 40 | } 41 | 42 | static void append_zeros(std::vector* buffer, std::size_t count) { 43 | buffer->insert(buffer->end(), count, '\0'); 44 | } 45 | 46 | static void resize(std::vector* buffer, std::size_t size) { 47 | protozero_assert(size < buffer->size()); 48 | buffer->resize(size); 49 | } 50 | 51 | static void reserve_additional(std::vector* buffer, std::size_t size) { 52 | buffer->reserve(buffer->size() + size); 53 | } 54 | 55 | static void erase_range(std::vector* buffer, std::size_t from, std::size_t to) { 56 | protozero_assert(from <= buffer->size()); 57 | protozero_assert(to <= buffer->size()); 58 | protozero_assert(from <= to); 59 | buffer->erase(std::next(buffer->begin(), from), std::next(buffer->begin(), to)); 60 | } 61 | 62 | static char* at_pos(std::vector* buffer, std::size_t pos) { 63 | protozero_assert(pos <= buffer->size()); 64 | return (&*buffer->begin()) + pos; 65 | } 66 | 67 | static void push_back(std::vector* buffer, char ch) { 68 | buffer->push_back(ch); 69 | } 70 | 71 | }; 72 | /// @endcond 73 | 74 | } // namespace protozero 75 | 76 | #endif // PROTOZERO_BUFFER_VECTOR_HPP 77 | -------------------------------------------------------------------------------- /contrib/protozero/1.7.0/include/protozero/byteswap.hpp: -------------------------------------------------------------------------------- 1 | #ifndef PROTOZERO_BYTESWAP_HPP 2 | #define PROTOZERO_BYTESWAP_HPP 3 | 4 | /***************************************************************************** 5 | 6 | protozero - Minimalistic protocol buffer decoder and encoder in C++. 7 | 8 | This file is from https://github.com/mapbox/protozero where you can find more 9 | documentation. 10 | 11 | *****************************************************************************/ 12 | 13 | /** 14 | * @file byteswap.hpp 15 | * 16 | * @brief Contains functions to swap bytes in values (for different endianness). 17 | */ 18 | 19 | #include "config.hpp" 20 | 21 | #include 22 | 23 | namespace protozero { 24 | namespace detail { 25 | 26 | inline uint32_t byteswap_impl(uint32_t value) noexcept { 27 | #ifdef PROTOZERO_USE_BUILTIN_BSWAP 28 | return __builtin_bswap32(value); 29 | #else 30 | return ((value & 0xff000000U) >> 24U) | 31 | ((value & 0x00ff0000U) >> 8U) | 32 | ((value & 0x0000ff00U) << 8U) | 33 | ((value & 0x000000ffU) << 24U); 34 | #endif 35 | } 36 | 37 | inline uint64_t byteswap_impl(uint64_t value) noexcept { 38 | #ifdef PROTOZERO_USE_BUILTIN_BSWAP 39 | return __builtin_bswap64(value); 40 | #else 41 | return ((value & 0xff00000000000000ULL) >> 56U) | 42 | ((value & 0x00ff000000000000ULL) >> 40U) | 43 | ((value & 0x0000ff0000000000ULL) >> 24U) | 44 | ((value & 0x000000ff00000000ULL) >> 8U) | 45 | ((value & 0x00000000ff000000ULL) << 8U) | 46 | ((value & 0x0000000000ff0000ULL) << 24U) | 47 | ((value & 0x000000000000ff00ULL) << 40U) | 48 | ((value & 0x00000000000000ffULL) << 56U); 49 | #endif 50 | } 51 | 52 | } // end namespace detail 53 | 54 | /// byteswap the data pointed to by ptr in-place. 55 | inline void byteswap_inplace(uint32_t* ptr) noexcept { 56 | *ptr = detail::byteswap_impl(*ptr); 57 | } 58 | 59 | /// byteswap the data pointed to by ptr in-place. 60 | inline void byteswap_inplace(uint64_t* ptr) noexcept { 61 | *ptr = detail::byteswap_impl(*ptr); 62 | } 63 | 64 | /// byteswap the data pointed to by ptr in-place. 65 | inline void byteswap_inplace(int32_t* ptr) noexcept { 66 | auto* bptr = reinterpret_cast(ptr); 67 | *bptr = detail::byteswap_impl(*bptr); 68 | } 69 | 70 | /// byteswap the data pointed to by ptr in-place. 71 | inline void byteswap_inplace(int64_t* ptr) noexcept { 72 | auto* bptr = reinterpret_cast(ptr); 73 | *bptr = detail::byteswap_impl(*bptr); 74 | } 75 | 76 | /// byteswap the data pointed to by ptr in-place. 77 | inline void byteswap_inplace(float* ptr) noexcept { 78 | auto* bptr = reinterpret_cast(ptr); 79 | *bptr = detail::byteswap_impl(*bptr); 80 | } 81 | 82 | /// byteswap the data pointed to by ptr in-place. 83 | inline void byteswap_inplace(double* ptr) noexcept { 84 | auto* bptr = reinterpret_cast(ptr); 85 | *bptr = detail::byteswap_impl(*bptr); 86 | } 87 | 88 | namespace detail { 89 | 90 | // Added for backwards compatibility with any code that might use this 91 | // function (even if it shouldn't have). Will be removed in a later 92 | // version of protozero. 93 | using ::protozero::byteswap_inplace; 94 | 95 | } // end namespace detail 96 | 97 | } // end namespace protozero 98 | 99 | #endif // PROTOZERO_BYTESWAP_HPP 100 | -------------------------------------------------------------------------------- /contrib/protozero/1.7.0/include/protozero/config.hpp: -------------------------------------------------------------------------------- 1 | #ifndef PROTOZERO_CONFIG_HPP 2 | #define PROTOZERO_CONFIG_HPP 3 | 4 | /***************************************************************************** 5 | 6 | protozero - Minimalistic protocol buffer decoder and encoder in C++. 7 | 8 | This file is from https://github.com/mapbox/protozero where you can find more 9 | documentation. 10 | 11 | *****************************************************************************/ 12 | 13 | #include 14 | 15 | /** 16 | * @file config.hpp 17 | * 18 | * @brief Contains macro checks for different configurations. 19 | */ 20 | 21 | #define PROTOZERO_LITTLE_ENDIAN 1234 22 | #define PROTOZERO_BIG_ENDIAN 4321 23 | 24 | // Find out which byte order the machine has. 25 | #if defined(__BYTE_ORDER) 26 | # if (__BYTE_ORDER == __LITTLE_ENDIAN) 27 | # define PROTOZERO_BYTE_ORDER PROTOZERO_LITTLE_ENDIAN 28 | # endif 29 | # if (__BYTE_ORDER == __BIG_ENDIAN) 30 | # define PROTOZERO_BYTE_ORDER PROTOZERO_BIG_ENDIAN 31 | # endif 32 | #else 33 | // This probably isn't a very good default, but might do until we figure 34 | // out something better. 35 | # define PROTOZERO_BYTE_ORDER PROTOZERO_LITTLE_ENDIAN 36 | #endif 37 | 38 | // Check whether __builtin_bswap is available 39 | #if defined(__GNUC__) || defined(__clang__) 40 | # define PROTOZERO_USE_BUILTIN_BSWAP 41 | #endif 42 | 43 | // Wrapper for assert() used for testing 44 | #ifndef protozero_assert 45 | # define protozero_assert(x) assert(x) 46 | #endif 47 | 48 | #endif // PROTOZERO_CONFIG_HPP 49 | -------------------------------------------------------------------------------- /contrib/protozero/1.7.0/include/protozero/exception.hpp: -------------------------------------------------------------------------------- 1 | #ifndef PROTOZERO_EXCEPTION_HPP 2 | #define PROTOZERO_EXCEPTION_HPP 3 | 4 | /***************************************************************************** 5 | 6 | protozero - Minimalistic protocol buffer decoder and encoder in C++. 7 | 8 | This file is from https://github.com/mapbox/protozero where you can find more 9 | documentation. 10 | 11 | *****************************************************************************/ 12 | 13 | /** 14 | * @file exception.hpp 15 | * 16 | * @brief Contains the exceptions used in the protozero library. 17 | */ 18 | 19 | #include 20 | 21 | /** 22 | * @brief All parts of the protozero header-only library are in this namespace. 23 | */ 24 | namespace protozero { 25 | 26 | /** 27 | * All exceptions explicitly thrown by the functions of the protozero library 28 | * derive from this exception. 29 | */ 30 | struct exception : std::exception { 31 | /// Returns the explanatory string. 32 | const char* what() const noexcept override { 33 | return "pbf exception"; 34 | } 35 | }; 36 | 37 | /** 38 | * This exception is thrown when parsing a varint thats larger than allowed. 39 | * This should never happen unless the data is corrupted. 40 | */ 41 | struct varint_too_long_exception : exception { 42 | /// Returns the explanatory string. 43 | const char* what() const noexcept override { 44 | return "varint too long exception"; 45 | } 46 | }; 47 | 48 | /** 49 | * This exception is thrown when the wire type of a pdf field is unknown. 50 | * This should never happen unless the data is corrupted. 51 | */ 52 | struct unknown_pbf_wire_type_exception : exception { 53 | /// Returns the explanatory string. 54 | const char* what() const noexcept override { 55 | return "unknown pbf field type exception"; 56 | } 57 | }; 58 | 59 | /** 60 | * This exception is thrown when we are trying to read a field and there 61 | * are not enough bytes left in the buffer to read it. Almost all functions 62 | * of the pbf_reader class can throw this exception. 63 | * 64 | * This should never happen unless the data is corrupted or you have 65 | * initialized the pbf_reader object with incomplete data. 66 | */ 67 | struct end_of_buffer_exception : exception { 68 | /// Returns the explanatory string. 69 | const char* what() const noexcept override { 70 | return "end of buffer exception"; 71 | } 72 | }; 73 | 74 | /** 75 | * This exception is thrown when a tag has an invalid value. Tags must be 76 | * unsigned integers between 1 and 2^29-1. Tags between 19000 and 19999 are 77 | * not allowed. See 78 | * https://developers.google.com/protocol-buffers/docs/proto#assigning-tags 79 | */ 80 | struct invalid_tag_exception : exception { 81 | /// Returns the explanatory string. 82 | const char* what() const noexcept override { 83 | return "invalid tag exception"; 84 | } 85 | }; 86 | 87 | /** 88 | * This exception is thrown when a length field of a packed repeated field is 89 | * invalid. For fixed size types the length must be a multiple of the size of 90 | * the type. 91 | */ 92 | struct invalid_length_exception : exception { 93 | /// Returns the explanatory string. 94 | const char* what() const noexcept override { 95 | return "invalid length exception"; 96 | } 97 | }; 98 | 99 | } // end namespace protozero 100 | 101 | #endif // PROTOZERO_EXCEPTION_HPP 102 | -------------------------------------------------------------------------------- /contrib/protozero/1.7.0/include/protozero/pbf_builder.hpp: -------------------------------------------------------------------------------- 1 | #ifndef PROTOZERO_PBF_BUILDER_HPP 2 | #define PROTOZERO_PBF_BUILDER_HPP 3 | 4 | /***************************************************************************** 5 | 6 | protozero - Minimalistic protocol buffer decoder and encoder in C++. 7 | 8 | This file is from https://github.com/mapbox/protozero where you can find more 9 | documentation. 10 | 11 | *****************************************************************************/ 12 | 13 | /** 14 | * @file pbf_builder.hpp 15 | * 16 | * @brief Contains the pbf_builder template class. 17 | */ 18 | 19 | #include "basic_pbf_builder.hpp" 20 | #include "pbf_writer.hpp" 21 | 22 | #include 23 | 24 | namespace protozero { 25 | 26 | /// Specialization of basic_pbf_builder using std::string as buffer type. 27 | template 28 | using pbf_builder = basic_pbf_builder; 29 | 30 | } // end namespace protozero 31 | 32 | #endif // PROTOZERO_PBF_BUILDER_HPP 33 | -------------------------------------------------------------------------------- /contrib/protozero/1.7.0/include/protozero/pbf_writer.hpp: -------------------------------------------------------------------------------- 1 | #ifndef PROTOZERO_PBF_WRITER_HPP 2 | #define PROTOZERO_PBF_WRITER_HPP 3 | 4 | /***************************************************************************** 5 | 6 | protozero - Minimalistic protocol buffer decoder and encoder in C++. 7 | 8 | This file is from https://github.com/mapbox/protozero where you can find more 9 | documentation. 10 | 11 | *****************************************************************************/ 12 | 13 | /** 14 | * @file pbf_writer.hpp 15 | * 16 | * @brief Contains the pbf_writer class. 17 | */ 18 | 19 | #include "basic_pbf_writer.hpp" 20 | #include "buffer_string.hpp" 21 | 22 | #include 23 | #include 24 | 25 | namespace protozero { 26 | 27 | /** 28 | * Specialization of basic_pbf_writer using std::string as buffer type. 29 | */ 30 | using pbf_writer = basic_pbf_writer; 31 | 32 | /// Class for generating packed repeated bool fields. 33 | using packed_field_bool = detail::packed_field_varint; 34 | 35 | /// Class for generating packed repeated enum fields. 36 | using packed_field_enum = detail::packed_field_varint; 37 | 38 | /// Class for generating packed repeated int32 fields. 39 | using packed_field_int32 = detail::packed_field_varint; 40 | 41 | /// Class for generating packed repeated sint32 fields. 42 | using packed_field_sint32 = detail::packed_field_svarint; 43 | 44 | /// Class for generating packed repeated uint32 fields. 45 | using packed_field_uint32 = detail::packed_field_varint; 46 | 47 | /// Class for generating packed repeated int64 fields. 48 | using packed_field_int64 = detail::packed_field_varint; 49 | 50 | /// Class for generating packed repeated sint64 fields. 51 | using packed_field_sint64 = detail::packed_field_svarint; 52 | 53 | /// Class for generating packed repeated uint64 fields. 54 | using packed_field_uint64 = detail::packed_field_varint; 55 | 56 | /// Class for generating packed repeated fixed32 fields. 57 | using packed_field_fixed32 = detail::packed_field_fixed; 58 | 59 | /// Class for generating packed repeated sfixed32 fields. 60 | using packed_field_sfixed32 = detail::packed_field_fixed; 61 | 62 | /// Class for generating packed repeated fixed64 fields. 63 | using packed_field_fixed64 = detail::packed_field_fixed; 64 | 65 | /// Class for generating packed repeated sfixed64 fields. 66 | using packed_field_sfixed64 = detail::packed_field_fixed; 67 | 68 | /// Class for generating packed repeated float fields. 69 | using packed_field_float = detail::packed_field_fixed; 70 | 71 | /// Class for generating packed repeated double fields. 72 | using packed_field_double = detail::packed_field_fixed; 73 | 74 | } // end namespace protozero 75 | 76 | #endif // PROTOZERO_PBF_WRITER_HPP 77 | -------------------------------------------------------------------------------- /contrib/protozero/1.7.0/include/protozero/types.hpp: -------------------------------------------------------------------------------- 1 | #ifndef PROTOZERO_TYPES_HPP 2 | #define PROTOZERO_TYPES_HPP 3 | 4 | /***************************************************************************** 5 | 6 | protozero - Minimalistic protocol buffer decoder and encoder in C++. 7 | 8 | This file is from https://github.com/mapbox/protozero where you can find more 9 | documentation. 10 | 11 | *****************************************************************************/ 12 | 13 | /** 14 | * @file types.hpp 15 | * 16 | * @brief Contains the declaration of low-level types used in the pbf format. 17 | */ 18 | 19 | #include "config.hpp" 20 | 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | 28 | namespace protozero { 29 | 30 | /** 31 | * The type used for field tags (field numbers). 32 | */ 33 | using pbf_tag_type = uint32_t; 34 | 35 | /** 36 | * The type used to encode type information. 37 | * See the table on 38 | * https://developers.google.com/protocol-buffers/docs/encoding 39 | */ 40 | enum class pbf_wire_type : uint32_t { 41 | varint = 0, // int32/64, uint32/64, sint32/64, bool, enum 42 | fixed64 = 1, // fixed64, sfixed64, double 43 | length_delimited = 2, // string, bytes, nested messages, packed repeated fields 44 | fixed32 = 5, // fixed32, sfixed32, float 45 | unknown = 99 // used for default setting in this library 46 | }; 47 | 48 | /** 49 | * Get the tag and wire type of the current field in one integer suitable 50 | * for comparison with a switch statement. 51 | * 52 | * See pbf_reader.tag_and_type() for an example how to use this. 53 | */ 54 | template 55 | constexpr inline uint32_t tag_and_type(T tag, pbf_wire_type wire_type) noexcept { 56 | return (static_cast(static_cast(tag)) << 3U) | static_cast(wire_type); 57 | } 58 | 59 | /** 60 | * The type used for length values, such as the length of a field. 61 | */ 62 | using pbf_length_type = uint32_t; 63 | 64 | } // end namespace protozero 65 | 66 | #endif // PROTOZERO_TYPES_HPP 67 | -------------------------------------------------------------------------------- /contrib/protozero/1.7.0/include/protozero/version.hpp: -------------------------------------------------------------------------------- 1 | #ifndef PROTOZERO_VERSION_HPP 2 | #define PROTOZERO_VERSION_HPP 3 | 4 | /***************************************************************************** 5 | 6 | protozero - Minimalistic protocol buffer decoder and encoder in C++. 7 | 8 | This file is from https://github.com/mapbox/protozero where you can find more 9 | documentation. 10 | 11 | *****************************************************************************/ 12 | 13 | /** 14 | * @file version.hpp 15 | * 16 | * @brief Contains macros defining the protozero version. 17 | */ 18 | 19 | /// The major version number 20 | #define PROTOZERO_VERSION_MAJOR 1 21 | 22 | /// The minor version number 23 | #define PROTOZERO_VERSION_MINOR 7 24 | 25 | /// The patch number 26 | #define PROTOZERO_VERSION_PATCH 0 27 | 28 | /// The complete version number 29 | #define PROTOZERO_VERSION_CODE (PROTOZERO_VERSION_MAJOR * 10000 + PROTOZERO_VERSION_MINOR * 100 + PROTOZERO_VERSION_PATCH) 30 | 31 | /// Version number as string 32 | #define PROTOZERO_VERSION_STRING "1.7.0" 33 | 34 | #endif // PROTOZERO_VERSION_HPP 35 | -------------------------------------------------------------------------------- /contrib/rapidjson/1.1.0/include/rapidjson/filereadstream.h: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making RapidJSON available. 2 | // 3 | // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. 4 | // 5 | // Licensed under the MIT License (the "License"); you may not use this file except 6 | // in compliance with the License. You may obtain a copy of the License at 7 | // 8 | // http://opensource.org/licenses/MIT 9 | // 10 | // Unless required by applicable law or agreed to in writing, software distributed 11 | // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 12 | // CONDITIONS OF ANY KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations under the License. 14 | 15 | #ifndef RAPIDJSON_FILEREADSTREAM_H_ 16 | #define RAPIDJSON_FILEREADSTREAM_H_ 17 | 18 | #include "stream.h" 19 | #include 20 | 21 | #ifdef __clang__ 22 | RAPIDJSON_DIAG_PUSH 23 | RAPIDJSON_DIAG_OFF(padded) 24 | RAPIDJSON_DIAG_OFF(unreachable-code) 25 | RAPIDJSON_DIAG_OFF(missing-noreturn) 26 | #endif 27 | 28 | RAPIDJSON_NAMESPACE_BEGIN 29 | 30 | //! File byte stream for input using fread(). 31 | /*! 32 | \note implements Stream concept 33 | */ 34 | class FileReadStream { 35 | public: 36 | typedef char Ch; //!< Character type (byte). 37 | 38 | //! Constructor. 39 | /*! 40 | \param fp File pointer opened for read. 41 | \param buffer user-supplied buffer. 42 | \param bufferSize size of buffer in bytes. Must >=4 bytes. 43 | */ 44 | FileReadStream(std::FILE* fp, char* buffer, size_t bufferSize) : fp_(fp), buffer_(buffer), bufferSize_(bufferSize), bufferLast_(0), current_(buffer_), readCount_(0), count_(0), eof_(false) { 45 | RAPIDJSON_ASSERT(fp_ != 0); 46 | RAPIDJSON_ASSERT(bufferSize >= 4); 47 | Read(); 48 | } 49 | 50 | Ch Peek() const { return *current_; } 51 | Ch Take() { Ch c = *current_; Read(); return c; } 52 | size_t Tell() const { return count_ + static_cast(current_ - buffer_); } 53 | 54 | // Not implemented 55 | void Put(Ch) { RAPIDJSON_ASSERT(false); } 56 | void Flush() { RAPIDJSON_ASSERT(false); } 57 | Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } 58 | size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } 59 | 60 | // For encoding detection only. 61 | const Ch* Peek4() const { 62 | return (current_ + 4 <= bufferLast_) ? current_ : 0; 63 | } 64 | 65 | private: 66 | void Read() { 67 | if (current_ < bufferLast_) 68 | ++current_; 69 | else if (!eof_) { 70 | count_ += readCount_; 71 | readCount_ = fread(buffer_, 1, bufferSize_, fp_); 72 | bufferLast_ = buffer_ + readCount_ - 1; 73 | current_ = buffer_; 74 | 75 | if (readCount_ < bufferSize_) { 76 | buffer_[readCount_] = '\0'; 77 | ++bufferLast_; 78 | eof_ = true; 79 | } 80 | } 81 | } 82 | 83 | std::FILE* fp_; 84 | Ch *buffer_; 85 | size_t bufferSize_; 86 | Ch *bufferLast_; 87 | Ch *current_; 88 | size_t readCount_; 89 | size_t count_; //!< Number of characters read 90 | bool eof_; 91 | }; 92 | 93 | RAPIDJSON_NAMESPACE_END 94 | 95 | #ifdef __clang__ 96 | RAPIDJSON_DIAG_POP 97 | #endif 98 | 99 | #endif // RAPIDJSON_FILESTREAM_H_ 100 | -------------------------------------------------------------------------------- /contrib/rapidjson/1.1.0/include/rapidjson/filewritestream.h: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making RapidJSON available. 2 | // 3 | // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. 4 | // 5 | // Licensed under the MIT License (the "License"); you may not use this file except 6 | // in compliance with the License. You may obtain a copy of the License at 7 | // 8 | // http://opensource.org/licenses/MIT 9 | // 10 | // Unless required by applicable law or agreed to in writing, software distributed 11 | // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 12 | // CONDITIONS OF ANY KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations under the License. 14 | 15 | #ifndef RAPIDJSON_FILEWRITESTREAM_H_ 16 | #define RAPIDJSON_FILEWRITESTREAM_H_ 17 | 18 | #include "stream.h" 19 | #include 20 | 21 | #ifdef __clang__ 22 | RAPIDJSON_DIAG_PUSH 23 | RAPIDJSON_DIAG_OFF(unreachable-code) 24 | #endif 25 | 26 | RAPIDJSON_NAMESPACE_BEGIN 27 | 28 | //! Wrapper of C file stream for input using fread(). 29 | /*! 30 | \note implements Stream concept 31 | */ 32 | class FileWriteStream { 33 | public: 34 | typedef char Ch; //!< Character type. Only support char. 35 | 36 | FileWriteStream(std::FILE* fp, char* buffer, size_t bufferSize) : fp_(fp), buffer_(buffer), bufferEnd_(buffer + bufferSize), current_(buffer_) { 37 | RAPIDJSON_ASSERT(fp_ != 0); 38 | } 39 | 40 | void Put(char c) { 41 | if (current_ >= bufferEnd_) 42 | Flush(); 43 | 44 | *current_++ = c; 45 | } 46 | 47 | void PutN(char c, size_t n) { 48 | size_t avail = static_cast(bufferEnd_ - current_); 49 | while (n > avail) { 50 | std::memset(current_, c, avail); 51 | current_ += avail; 52 | Flush(); 53 | n -= avail; 54 | avail = static_cast(bufferEnd_ - current_); 55 | } 56 | 57 | if (n > 0) { 58 | std::memset(current_, c, n); 59 | current_ += n; 60 | } 61 | } 62 | 63 | void Flush() { 64 | if (current_ != buffer_) { 65 | size_t result = fwrite(buffer_, 1, static_cast(current_ - buffer_), fp_); 66 | if (result < static_cast(current_ - buffer_)) { 67 | // failure deliberately ignored at this time 68 | // added to avoid warn_unused_result build errors 69 | } 70 | current_ = buffer_; 71 | } 72 | } 73 | 74 | // Not implemented 75 | char Peek() const { RAPIDJSON_ASSERT(false); return 0; } 76 | char Take() { RAPIDJSON_ASSERT(false); return 0; } 77 | size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; } 78 | char* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } 79 | size_t PutEnd(char*) { RAPIDJSON_ASSERT(false); return 0; } 80 | 81 | private: 82 | // Prohibit copy constructor & assignment operator. 83 | FileWriteStream(const FileWriteStream&); 84 | FileWriteStream& operator=(const FileWriteStream&); 85 | 86 | std::FILE* fp_; 87 | char *buffer_; 88 | char *bufferEnd_; 89 | char *current_; 90 | }; 91 | 92 | //! Implement specialized version of PutN() with memset() for better performance. 93 | template<> 94 | inline void PutN(FileWriteStream& stream, char c, size_t n) { 95 | stream.PutN(c, n); 96 | } 97 | 98 | RAPIDJSON_NAMESPACE_END 99 | 100 | #ifdef __clang__ 101 | RAPIDJSON_DIAG_POP 102 | #endif 103 | 104 | #endif // RAPIDJSON_FILESTREAM_H_ 105 | -------------------------------------------------------------------------------- /contrib/rapidjson/1.1.0/include/rapidjson/internal/ieee754.h: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making RapidJSON available. 2 | // 3 | // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. 4 | // 5 | // Licensed under the MIT License (the "License"); you may not use this file except 6 | // in compliance with the License. You may obtain a copy of the License at 7 | // 8 | // http://opensource.org/licenses/MIT 9 | // 10 | // Unless required by applicable law or agreed to in writing, software distributed 11 | // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 12 | // CONDITIONS OF ANY KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations under the License. 14 | 15 | #ifndef RAPIDJSON_IEEE754_ 16 | #define RAPIDJSON_IEEE754_ 17 | 18 | #include "../rapidjson.h" 19 | 20 | RAPIDJSON_NAMESPACE_BEGIN 21 | namespace internal { 22 | 23 | class Double { 24 | public: 25 | Double() {} 26 | Double(double d) : d_(d) {} 27 | Double(uint64_t u) : u_(u) {} 28 | 29 | double Value() const { return d_; } 30 | uint64_t Uint64Value() const { return u_; } 31 | 32 | double NextPositiveDouble() const { 33 | RAPIDJSON_ASSERT(!Sign()); 34 | return Double(u_ + 1).Value(); 35 | } 36 | 37 | bool Sign() const { return (u_ & kSignMask) != 0; } 38 | uint64_t Significand() const { return u_ & kSignificandMask; } 39 | int Exponent() const { return static_cast(((u_ & kExponentMask) >> kSignificandSize) - kExponentBias); } 40 | 41 | bool IsNan() const { return (u_ & kExponentMask) == kExponentMask && Significand() != 0; } 42 | bool IsInf() const { return (u_ & kExponentMask) == kExponentMask && Significand() == 0; } 43 | bool IsNanOrInf() const { return (u_ & kExponentMask) == kExponentMask; } 44 | bool IsNormal() const { return (u_ & kExponentMask) != 0 || Significand() == 0; } 45 | bool IsZero() const { return (u_ & (kExponentMask | kSignificandMask)) == 0; } 46 | 47 | uint64_t IntegerSignificand() const { return IsNormal() ? Significand() | kHiddenBit : Significand(); } 48 | int IntegerExponent() const { return (IsNormal() ? Exponent() : kDenormalExponent) - kSignificandSize; } 49 | uint64_t ToBias() const { return (u_ & kSignMask) ? ~u_ + 1 : u_ | kSignMask; } 50 | 51 | static unsigned EffectiveSignificandSize(int order) { 52 | if (order >= -1021) 53 | return 53; 54 | else if (order <= -1074) 55 | return 0; 56 | else 57 | return static_cast(order) + 1074; 58 | } 59 | 60 | private: 61 | static const int kSignificandSize = 52; 62 | static const int kExponentBias = 0x3FF; 63 | static const int kDenormalExponent = 1 - kExponentBias; 64 | static const uint64_t kSignMask = RAPIDJSON_UINT64_C2(0x80000000, 0x00000000); 65 | static const uint64_t kExponentMask = RAPIDJSON_UINT64_C2(0x7FF00000, 0x00000000); 66 | static const uint64_t kSignificandMask = RAPIDJSON_UINT64_C2(0x000FFFFF, 0xFFFFFFFF); 67 | static const uint64_t kHiddenBit = RAPIDJSON_UINT64_C2(0x00100000, 0x00000000); 68 | 69 | union { 70 | double d_; 71 | uint64_t u_; 72 | }; 73 | }; 74 | 75 | } // namespace internal 76 | RAPIDJSON_NAMESPACE_END 77 | 78 | #endif // RAPIDJSON_IEEE754_ 79 | -------------------------------------------------------------------------------- /contrib/rapidjson/1.1.0/include/rapidjson/internal/pow10.h: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making RapidJSON available. 2 | // 3 | // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. 4 | // 5 | // Licensed under the MIT License (the "License"); you may not use this file except 6 | // in compliance with the License. You may obtain a copy of the License at 7 | // 8 | // http://opensource.org/licenses/MIT 9 | // 10 | // Unless required by applicable law or agreed to in writing, software distributed 11 | // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 12 | // CONDITIONS OF ANY KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations under the License. 14 | 15 | #ifndef RAPIDJSON_POW10_ 16 | #define RAPIDJSON_POW10_ 17 | 18 | #include "../rapidjson.h" 19 | 20 | RAPIDJSON_NAMESPACE_BEGIN 21 | namespace internal { 22 | 23 | //! Computes integer powers of 10 in double (10.0^n). 24 | /*! This function uses lookup table for fast and accurate results. 25 | \param n non-negative exponent. Must <= 308. 26 | \return 10.0^n 27 | */ 28 | inline double Pow10(int n) { 29 | static const double e[] = { // 1e-0...1e308: 309 * 8 bytes = 2472 bytes 30 | 1e+0, 31 | 1e+1, 1e+2, 1e+3, 1e+4, 1e+5, 1e+6, 1e+7, 1e+8, 1e+9, 1e+10, 1e+11, 1e+12, 1e+13, 1e+14, 1e+15, 1e+16, 1e+17, 1e+18, 1e+19, 1e+20, 32 | 1e+21, 1e+22, 1e+23, 1e+24, 1e+25, 1e+26, 1e+27, 1e+28, 1e+29, 1e+30, 1e+31, 1e+32, 1e+33, 1e+34, 1e+35, 1e+36, 1e+37, 1e+38, 1e+39, 1e+40, 33 | 1e+41, 1e+42, 1e+43, 1e+44, 1e+45, 1e+46, 1e+47, 1e+48, 1e+49, 1e+50, 1e+51, 1e+52, 1e+53, 1e+54, 1e+55, 1e+56, 1e+57, 1e+58, 1e+59, 1e+60, 34 | 1e+61, 1e+62, 1e+63, 1e+64, 1e+65, 1e+66, 1e+67, 1e+68, 1e+69, 1e+70, 1e+71, 1e+72, 1e+73, 1e+74, 1e+75, 1e+76, 1e+77, 1e+78, 1e+79, 1e+80, 35 | 1e+81, 1e+82, 1e+83, 1e+84, 1e+85, 1e+86, 1e+87, 1e+88, 1e+89, 1e+90, 1e+91, 1e+92, 1e+93, 1e+94, 1e+95, 1e+96, 1e+97, 1e+98, 1e+99, 1e+100, 36 | 1e+101,1e+102,1e+103,1e+104,1e+105,1e+106,1e+107,1e+108,1e+109,1e+110,1e+111,1e+112,1e+113,1e+114,1e+115,1e+116,1e+117,1e+118,1e+119,1e+120, 37 | 1e+121,1e+122,1e+123,1e+124,1e+125,1e+126,1e+127,1e+128,1e+129,1e+130,1e+131,1e+132,1e+133,1e+134,1e+135,1e+136,1e+137,1e+138,1e+139,1e+140, 38 | 1e+141,1e+142,1e+143,1e+144,1e+145,1e+146,1e+147,1e+148,1e+149,1e+150,1e+151,1e+152,1e+153,1e+154,1e+155,1e+156,1e+157,1e+158,1e+159,1e+160, 39 | 1e+161,1e+162,1e+163,1e+164,1e+165,1e+166,1e+167,1e+168,1e+169,1e+170,1e+171,1e+172,1e+173,1e+174,1e+175,1e+176,1e+177,1e+178,1e+179,1e+180, 40 | 1e+181,1e+182,1e+183,1e+184,1e+185,1e+186,1e+187,1e+188,1e+189,1e+190,1e+191,1e+192,1e+193,1e+194,1e+195,1e+196,1e+197,1e+198,1e+199,1e+200, 41 | 1e+201,1e+202,1e+203,1e+204,1e+205,1e+206,1e+207,1e+208,1e+209,1e+210,1e+211,1e+212,1e+213,1e+214,1e+215,1e+216,1e+217,1e+218,1e+219,1e+220, 42 | 1e+221,1e+222,1e+223,1e+224,1e+225,1e+226,1e+227,1e+228,1e+229,1e+230,1e+231,1e+232,1e+233,1e+234,1e+235,1e+236,1e+237,1e+238,1e+239,1e+240, 43 | 1e+241,1e+242,1e+243,1e+244,1e+245,1e+246,1e+247,1e+248,1e+249,1e+250,1e+251,1e+252,1e+253,1e+254,1e+255,1e+256,1e+257,1e+258,1e+259,1e+260, 44 | 1e+261,1e+262,1e+263,1e+264,1e+265,1e+266,1e+267,1e+268,1e+269,1e+270,1e+271,1e+272,1e+273,1e+274,1e+275,1e+276,1e+277,1e+278,1e+279,1e+280, 45 | 1e+281,1e+282,1e+283,1e+284,1e+285,1e+286,1e+287,1e+288,1e+289,1e+290,1e+291,1e+292,1e+293,1e+294,1e+295,1e+296,1e+297,1e+298,1e+299,1e+300, 46 | 1e+301,1e+302,1e+303,1e+304,1e+305,1e+306,1e+307,1e+308 47 | }; 48 | RAPIDJSON_ASSERT(n >= 0 && n <= 308); 49 | return e[n]; 50 | } 51 | 52 | } // namespace internal 53 | RAPIDJSON_NAMESPACE_END 54 | 55 | #endif // RAPIDJSON_POW10_ 56 | -------------------------------------------------------------------------------- /contrib/rapidjson/1.1.0/include/rapidjson/internal/strfunc.h: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making RapidJSON available. 2 | // 3 | // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. 4 | // 5 | // Licensed under the MIT License (the "License"); you may not use this file except 6 | // in compliance with the License. You may obtain a copy of the License at 7 | // 8 | // http://opensource.org/licenses/MIT 9 | // 10 | // Unless required by applicable law or agreed to in writing, software distributed 11 | // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 12 | // CONDITIONS OF ANY KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations under the License. 14 | 15 | #ifndef RAPIDJSON_INTERNAL_STRFUNC_H_ 16 | #define RAPIDJSON_INTERNAL_STRFUNC_H_ 17 | 18 | #include "../stream.h" 19 | 20 | RAPIDJSON_NAMESPACE_BEGIN 21 | namespace internal { 22 | 23 | //! Custom strlen() which works on different character types. 24 | /*! \tparam Ch Character type (e.g. char, wchar_t, short) 25 | \param s Null-terminated input string. 26 | \return Number of characters in the string. 27 | \note This has the same semantics as strlen(), the return value is not number of Unicode codepoints. 28 | */ 29 | template 30 | inline SizeType StrLen(const Ch* s) { 31 | const Ch* p = s; 32 | while (*p) ++p; 33 | return SizeType(p - s); 34 | } 35 | 36 | //! Returns number of code points in a encoded string. 37 | template 38 | bool CountStringCodePoint(const typename Encoding::Ch* s, SizeType length, SizeType* outCount) { 39 | GenericStringStream is(s); 40 | const typename Encoding::Ch* end = s + length; 41 | SizeType count = 0; 42 | while (is.src_ < end) { 43 | unsigned codepoint; 44 | if (!Encoding::Decode(is, &codepoint)) 45 | return false; 46 | count++; 47 | } 48 | *outCount = count; 49 | return true; 50 | } 51 | 52 | } // namespace internal 53 | RAPIDJSON_NAMESPACE_END 54 | 55 | #endif // RAPIDJSON_INTERNAL_STRFUNC_H_ 56 | -------------------------------------------------------------------------------- /contrib/rapidjson/1.1.0/include/rapidjson/internal/swap.h: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making RapidJSON available. 2 | // 3 | // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. 4 | // 5 | // Licensed under the MIT License (the "License"); you may not use this file except 6 | // in compliance with the License. You may obtain a copy of the License at 7 | // 8 | // http://opensource.org/licenses/MIT 9 | // 10 | // Unless required by applicable law or agreed to in writing, software distributed 11 | // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 12 | // CONDITIONS OF ANY KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations under the License. 14 | 15 | #ifndef RAPIDJSON_INTERNAL_SWAP_H_ 16 | #define RAPIDJSON_INTERNAL_SWAP_H_ 17 | 18 | #include "../rapidjson.h" 19 | 20 | #if defined(__clang__) 21 | RAPIDJSON_DIAG_PUSH 22 | RAPIDJSON_DIAG_OFF(c++98-compat) 23 | #endif 24 | 25 | RAPIDJSON_NAMESPACE_BEGIN 26 | namespace internal { 27 | 28 | //! Custom swap() to avoid dependency on C++ header 29 | /*! \tparam T Type of the arguments to swap, should be instantiated with primitive C++ types only. 30 | \note This has the same semantics as std::swap(). 31 | */ 32 | template 33 | inline void Swap(T& a, T& b) RAPIDJSON_NOEXCEPT { 34 | T tmp = a; 35 | a = b; 36 | b = tmp; 37 | } 38 | 39 | } // namespace internal 40 | RAPIDJSON_NAMESPACE_END 41 | 42 | #if defined(__clang__) 43 | RAPIDJSON_DIAG_POP 44 | #endif 45 | 46 | #endif // RAPIDJSON_INTERNAL_SWAP_H_ 47 | -------------------------------------------------------------------------------- /contrib/rapidjson/1.1.0/include/rapidjson/memorybuffer.h: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making RapidJSON available. 2 | // 3 | // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. 4 | // 5 | // Licensed under the MIT License (the "License"); you may not use this file except 6 | // in compliance with the License. You may obtain a copy of the License at 7 | // 8 | // http://opensource.org/licenses/MIT 9 | // 10 | // Unless required by applicable law or agreed to in writing, software distributed 11 | // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 12 | // CONDITIONS OF ANY KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations under the License. 14 | 15 | #ifndef RAPIDJSON_MEMORYBUFFER_H_ 16 | #define RAPIDJSON_MEMORYBUFFER_H_ 17 | 18 | #include "stream.h" 19 | #include "internal/stack.h" 20 | 21 | RAPIDJSON_NAMESPACE_BEGIN 22 | 23 | //! Represents an in-memory output byte stream. 24 | /*! 25 | This class is mainly for being wrapped by EncodedOutputStream or AutoUTFOutputStream. 26 | 27 | It is similar to FileWriteBuffer but the destination is an in-memory buffer instead of a file. 28 | 29 | Differences between MemoryBuffer and StringBuffer: 30 | 1. StringBuffer has Encoding but MemoryBuffer is only a byte buffer. 31 | 2. StringBuffer::GetString() returns a null-terminated string. MemoryBuffer::GetBuffer() returns a buffer without terminator. 32 | 33 | \tparam Allocator type for allocating memory buffer. 34 | \note implements Stream concept 35 | */ 36 | template 37 | struct GenericMemoryBuffer { 38 | typedef char Ch; // byte 39 | 40 | GenericMemoryBuffer(Allocator* allocator = 0, size_t capacity = kDefaultCapacity) : stack_(allocator, capacity) {} 41 | 42 | void Put(Ch c) { *stack_.template Push() = c; } 43 | void Flush() {} 44 | 45 | void Clear() { stack_.Clear(); } 46 | void ShrinkToFit() { stack_.ShrinkToFit(); } 47 | Ch* Push(size_t count) { return stack_.template Push(count); } 48 | void Pop(size_t count) { stack_.template Pop(count); } 49 | 50 | const Ch* GetBuffer() const { 51 | return stack_.template Bottom(); 52 | } 53 | 54 | size_t GetSize() const { return stack_.GetSize(); } 55 | 56 | static const size_t kDefaultCapacity = 256; 57 | mutable internal::Stack stack_; 58 | }; 59 | 60 | typedef GenericMemoryBuffer<> MemoryBuffer; 61 | 62 | //! Implement specialized version of PutN() with memset() for better performance. 63 | template<> 64 | inline void PutN(MemoryBuffer& memoryBuffer, char c, size_t n) { 65 | std::memset(memoryBuffer.stack_.Push(n), c, n * sizeof(c)); 66 | } 67 | 68 | RAPIDJSON_NAMESPACE_END 69 | 70 | #endif // RAPIDJSON_MEMORYBUFFER_H_ 71 | -------------------------------------------------------------------------------- /contrib/rapidjson/1.1.0/include/rapidjson/memorystream.h: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making RapidJSON available. 2 | // 3 | // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. 4 | // 5 | // Licensed under the MIT License (the "License"); you may not use this file except 6 | // in compliance with the License. You may obtain a copy of the License at 7 | // 8 | // http://opensource.org/licenses/MIT 9 | // 10 | // Unless required by applicable law or agreed to in writing, software distributed 11 | // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 12 | // CONDITIONS OF ANY KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations under the License. 14 | 15 | #ifndef RAPIDJSON_MEMORYSTREAM_H_ 16 | #define RAPIDJSON_MEMORYSTREAM_H_ 17 | 18 | #include "stream.h" 19 | 20 | #ifdef __clang__ 21 | RAPIDJSON_DIAG_PUSH 22 | RAPIDJSON_DIAG_OFF(unreachable-code) 23 | RAPIDJSON_DIAG_OFF(missing-noreturn) 24 | #endif 25 | 26 | RAPIDJSON_NAMESPACE_BEGIN 27 | 28 | //! Represents an in-memory input byte stream. 29 | /*! 30 | This class is mainly for being wrapped by EncodedInputStream or AutoUTFInputStream. 31 | 32 | It is similar to FileReadBuffer but the source is an in-memory buffer instead of a file. 33 | 34 | Differences between MemoryStream and StringStream: 35 | 1. StringStream has encoding but MemoryStream is a byte stream. 36 | 2. MemoryStream needs size of the source buffer and the buffer don't need to be null terminated. StringStream assume null-terminated string as source. 37 | 3. MemoryStream supports Peek4() for encoding detection. StringStream is specified with an encoding so it should not have Peek4(). 38 | \note implements Stream concept 39 | */ 40 | struct MemoryStream { 41 | typedef char Ch; // byte 42 | 43 | MemoryStream(const Ch *src, size_t size) : src_(src), begin_(src), end_(src + size), size_(size) {} 44 | 45 | Ch Peek() const { return RAPIDJSON_UNLIKELY(src_ == end_) ? '\0' : *src_; } 46 | Ch Take() { return RAPIDJSON_UNLIKELY(src_ == end_) ? '\0' : *src_++; } 47 | size_t Tell() const { return static_cast(src_ - begin_); } 48 | 49 | Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } 50 | void Put(Ch) { RAPIDJSON_ASSERT(false); } 51 | void Flush() { RAPIDJSON_ASSERT(false); } 52 | size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } 53 | 54 | // For encoding detection only. 55 | const Ch* Peek4() const { 56 | return Tell() + 4 <= size_ ? src_ : 0; 57 | } 58 | 59 | const Ch* src_; //!< Current read position. 60 | const Ch* begin_; //!< Original head of the string. 61 | const Ch* end_; //!< End of stream. 62 | size_t size_; //!< Size of the stream. 63 | }; 64 | 65 | RAPIDJSON_NAMESPACE_END 66 | 67 | #ifdef __clang__ 68 | RAPIDJSON_DIAG_POP 69 | #endif 70 | 71 | #endif // RAPIDJSON_MEMORYBUFFER_H_ 72 | -------------------------------------------------------------------------------- /contrib/rapidjson/1.1.0/include/rapidjson/ostreamwrapper.h: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making RapidJSON available. 2 | // 3 | // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. 4 | // 5 | // Licensed under the MIT License (the "License"); you may not use this file except 6 | // in compliance with the License. You may obtain a copy of the License at 7 | // 8 | // http://opensource.org/licenses/MIT 9 | // 10 | // Unless required by applicable law or agreed to in writing, software distributed 11 | // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 12 | // CONDITIONS OF ANY KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations under the License. 14 | 15 | #ifndef RAPIDJSON_OSTREAMWRAPPER_H_ 16 | #define RAPIDJSON_OSTREAMWRAPPER_H_ 17 | 18 | #include "stream.h" 19 | #include 20 | 21 | #ifdef __clang__ 22 | RAPIDJSON_DIAG_PUSH 23 | RAPIDJSON_DIAG_OFF(padded) 24 | #endif 25 | 26 | RAPIDJSON_NAMESPACE_BEGIN 27 | 28 | //! Wrapper of \c std::basic_ostream into RapidJSON's Stream concept. 29 | /*! 30 | The classes can be wrapped including but not limited to: 31 | 32 | - \c std::ostringstream 33 | - \c std::stringstream 34 | - \c std::wpstringstream 35 | - \c std::wstringstream 36 | - \c std::ifstream 37 | - \c std::fstream 38 | - \c std::wofstream 39 | - \c std::wfstream 40 | 41 | \tparam StreamType Class derived from \c std::basic_ostream. 42 | */ 43 | 44 | template 45 | class BasicOStreamWrapper { 46 | public: 47 | typedef typename StreamType::char_type Ch; 48 | BasicOStreamWrapper(StreamType& stream) : stream_(stream) {} 49 | 50 | void Put(Ch c) { 51 | stream_.put(c); 52 | } 53 | 54 | void Flush() { 55 | stream_.flush(); 56 | } 57 | 58 | // Not implemented 59 | char Peek() const { RAPIDJSON_ASSERT(false); return 0; } 60 | char Take() { RAPIDJSON_ASSERT(false); return 0; } 61 | size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; } 62 | char* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } 63 | size_t PutEnd(char*) { RAPIDJSON_ASSERT(false); return 0; } 64 | 65 | private: 66 | BasicOStreamWrapper(const BasicOStreamWrapper&); 67 | BasicOStreamWrapper& operator=(const BasicOStreamWrapper&); 68 | 69 | StreamType& stream_; 70 | }; 71 | 72 | typedef BasicOStreamWrapper OStreamWrapper; 73 | typedef BasicOStreamWrapper WOStreamWrapper; 74 | 75 | #ifdef __clang__ 76 | RAPIDJSON_DIAG_POP 77 | #endif 78 | 79 | RAPIDJSON_NAMESPACE_END 80 | 81 | #endif // RAPIDJSON_OSTREAMWRAPPER_H_ 82 | -------------------------------------------------------------------------------- /contrib/supercluster/0.3.2/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016, Mapbox 2 | 3 | Permission to use, copy, modify, and/or distribute this software for any 4 | purpose with or without fee is hereby granted, provided that the above 5 | copyright notice and this permission notice appear in all copies. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 8 | WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 9 | MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 10 | ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 11 | WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 12 | ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 13 | OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 14 | -------------------------------------------------------------------------------- /contrib/supercluster/0.3.2/README.md: -------------------------------------------------------------------------------- 1 | A C++14 port of [supercluster](https://github.com/mapbox/supercluster), a fast 2D point clustering library for use in interactive maps. 2 | 3 | [![Build Status](https://travis-ci.org/mapbox/supercluster.hpp.svg?branch=master)](https://travis-ci.org/mapbox/supercluster.hpp) 4 | -------------------------------------------------------------------------------- /contrib/variant/1.2.0/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) MapBox 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, 5 | are permitted provided that the following conditions are met: 6 | 7 | - Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | - Redistributions in binary form must reproduce the above copyright notice, this 10 | list of conditions and the following disclaimer in the documentation and/or 11 | other materials provided with the distribution. 12 | - Neither the name "MapBox" nor the names of its contributors may be 13 | used to endorse or promote products derived from this software without 14 | specific prior written permission. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 17 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR 20 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 23 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 25 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /contrib/variant/1.2.0/include/mapbox/optional.hpp: -------------------------------------------------------------------------------- 1 | #ifndef MAPBOX_UTIL_OPTIONAL_HPP 2 | #define MAPBOX_UTIL_OPTIONAL_HPP 3 | 4 | #pragma message("This implementation of optional is deprecated. See https://github.com/mapbox/variant/issues/64.") 5 | 6 | #include 7 | #include 8 | 9 | #include 10 | 11 | namespace mapbox { 12 | namespace util { 13 | 14 | template 15 | class optional 16 | { 17 | static_assert(!std::is_reference::value, "optional doesn't support references"); 18 | 19 | struct none_type 20 | { 21 | }; 22 | 23 | variant variant_; 24 | 25 | public: 26 | optional() = default; 27 | 28 | optional(optional const& rhs) 29 | { 30 | if (this != &rhs) 31 | { // protect against invalid self-assignment 32 | variant_ = rhs.variant_; 33 | } 34 | } 35 | 36 | optional(T const& v) { variant_ = v; } 37 | 38 | explicit operator bool() const noexcept { return variant_.template is(); } 39 | 40 | T const& get() const { return variant_.template get(); } 41 | T& get() { return variant_.template get(); } 42 | 43 | T const& operator*() const { return this->get(); } 44 | T operator*() { return this->get(); } 45 | 46 | optional& operator=(T const& v) 47 | { 48 | variant_ = v; 49 | return *this; 50 | } 51 | 52 | optional& operator=(optional const& rhs) 53 | { 54 | if (this != &rhs) 55 | { 56 | variant_ = rhs.variant_; 57 | } 58 | return *this; 59 | } 60 | 61 | template 62 | void emplace(Args&&... args) 63 | { 64 | variant_ = T{std::forward(args)...}; 65 | } 66 | 67 | void reset() { variant_ = none_type{}; } 68 | 69 | }; // class optional 70 | 71 | } // namespace util 72 | } // namespace mapbox 73 | 74 | #endif // MAPBOX_UTIL_OPTIONAL_HPP 75 | -------------------------------------------------------------------------------- /contrib/variant/1.2.0/include/mapbox/recursive_wrapper.hpp: -------------------------------------------------------------------------------- 1 | #ifndef MAPBOX_UTIL_RECURSIVE_WRAPPER_HPP 2 | #define MAPBOX_UTIL_RECURSIVE_WRAPPER_HPP 3 | 4 | // Based on variant/recursive_wrapper.hpp from boost. 5 | // 6 | // Original license: 7 | // 8 | // Copyright (c) 2002-2003 9 | // Eric Friedman, Itay Maman 10 | // 11 | // Distributed under the Boost Software License, Version 1.0. (See 12 | // accompanying file LICENSE_1_0.txt or copy at 13 | // http://www.boost.org/LICENSE_1_0.txt) 14 | 15 | #include 16 | #include 17 | 18 | namespace mapbox { 19 | namespace util { 20 | 21 | template 22 | class recursive_wrapper 23 | { 24 | 25 | T* p_; 26 | 27 | void assign(T const& rhs) 28 | { 29 | this->get() = rhs; 30 | } 31 | 32 | public: 33 | using type = T; 34 | 35 | /** 36 | * Default constructor default initializes the internally stored value. 37 | * For POD types this means nothing is done and the storage is 38 | * uninitialized. 39 | * 40 | * @throws std::bad_alloc if there is insufficient memory for an object 41 | * of type T. 42 | * @throws any exception thrown by the default constructur of T. 43 | */ 44 | recursive_wrapper() 45 | : p_(new T){} 46 | 47 | ~recursive_wrapper() noexcept { delete p_; } 48 | 49 | recursive_wrapper(recursive_wrapper const& operand) 50 | : p_(new T(operand.get())) {} 51 | 52 | recursive_wrapper(T const& operand) 53 | : p_(new T(operand)) {} 54 | 55 | recursive_wrapper(recursive_wrapper&& operand) 56 | : p_(new T(std::move(operand.get()))) {} 57 | 58 | recursive_wrapper(T&& operand) 59 | : p_(new T(std::move(operand))) {} 60 | 61 | inline recursive_wrapper& operator=(recursive_wrapper const& rhs) 62 | { 63 | assign(rhs.get()); 64 | return *this; 65 | } 66 | 67 | inline recursive_wrapper& operator=(T const& rhs) 68 | { 69 | assign(rhs); 70 | return *this; 71 | } 72 | 73 | inline void swap(recursive_wrapper& operand) noexcept 74 | { 75 | T* temp = operand.p_; 76 | operand.p_ = p_; 77 | p_ = temp; 78 | } 79 | 80 | recursive_wrapper& operator=(recursive_wrapper&& rhs) noexcept 81 | { 82 | swap(rhs); 83 | return *this; 84 | } 85 | 86 | recursive_wrapper& operator=(T&& rhs) 87 | { 88 | get() = std::move(rhs); 89 | return *this; 90 | } 91 | 92 | T& get() 93 | { 94 | assert(p_); 95 | return *get_pointer(); 96 | } 97 | 98 | T const& get() const 99 | { 100 | assert(p_); 101 | return *get_pointer(); 102 | } 103 | 104 | T* get_pointer() { return p_; } 105 | 106 | const T* get_pointer() const { return p_; } 107 | 108 | operator T const&() const { return this->get(); } 109 | 110 | operator T&() { return this->get(); } 111 | 112 | }; // class recursive_wrapper 113 | 114 | template 115 | inline void swap(recursive_wrapper& lhs, recursive_wrapper& rhs) noexcept 116 | { 117 | lhs.swap(rhs); 118 | } 119 | } // namespace util 120 | } // namespace mapbox 121 | 122 | #endif // MAPBOX_UTIL_RECURSIVE_WRAPPER_HPP 123 | -------------------------------------------------------------------------------- /contrib/variant/1.2.0/include/mapbox/variant_cast.hpp: -------------------------------------------------------------------------------- 1 | #ifndef VARIANT_CAST_HPP 2 | #define VARIANT_CAST_HPP 3 | 4 | #include 5 | 6 | namespace mapbox { 7 | namespace util { 8 | 9 | namespace detail { 10 | 11 | template 12 | class static_caster 13 | { 14 | public: 15 | template 16 | T& operator()(V& v) const 17 | { 18 | return static_cast(v); 19 | } 20 | }; 21 | 22 | template 23 | class dynamic_caster 24 | { 25 | public: 26 | using result_type = T&; 27 | template 28 | T& operator()(V& v, typename std::enable_if::value>::type* = nullptr) const 29 | { 30 | throw std::bad_cast(); 31 | } 32 | template 33 | T& operator()(V& v, typename std::enable_if::value>::type* = nullptr) const 34 | { 35 | return dynamic_cast(v); 36 | } 37 | }; 38 | 39 | template 40 | class dynamic_caster 41 | { 42 | public: 43 | using result_type = T*; 44 | template 45 | T* operator()(V& v, typename std::enable_if::value>::type* = nullptr) const 46 | { 47 | return nullptr; 48 | } 49 | template 50 | T* operator()(V& v, typename std::enable_if::value>::type* = nullptr) const 51 | { 52 | return dynamic_cast(&v); 53 | } 54 | }; 55 | } 56 | 57 | template 58 | typename detail::dynamic_caster::result_type 59 | dynamic_variant_cast(V& v) 60 | { 61 | return mapbox::util::apply_visitor(detail::dynamic_caster(), v); 62 | } 63 | 64 | template 65 | typename detail::dynamic_caster::result_type 66 | dynamic_variant_cast(const V& v) 67 | { 68 | return mapbox::util::apply_visitor(detail::dynamic_caster(), v); 69 | } 70 | 71 | template 72 | T& static_variant_cast(V& v) 73 | { 74 | return mapbox::util::apply_visitor(detail::static_caster(), v); 75 | } 76 | 77 | template 78 | const T& static_variant_cast(const V& v) 79 | { 80 | return mapbox::util::apply_visitor(detail::static_caster(), v); 81 | } 82 | } 83 | } 84 | 85 | #endif // VARIANT_CAST_HPP 86 | -------------------------------------------------------------------------------- /contrib/variant/1.2.0/include/mapbox/variant_io.hpp: -------------------------------------------------------------------------------- 1 | #ifndef MAPBOX_UTIL_VARIANT_IO_HPP 2 | #define MAPBOX_UTIL_VARIANT_IO_HPP 3 | 4 | #include 5 | 6 | #include 7 | 8 | namespace mapbox { 9 | namespace util { 10 | 11 | namespace detail { 12 | // operator<< helper 13 | template 14 | class printer 15 | { 16 | public: 17 | explicit printer(Out& out) 18 | : out_(out) {} 19 | printer& operator=(printer const&) = delete; 20 | 21 | // visitor 22 | template 23 | void operator()(T const& operand) const 24 | { 25 | out_ << operand; 26 | } 27 | 28 | private: 29 | Out& out_; 30 | }; 31 | } 32 | 33 | // operator<< 34 | template 35 | VARIANT_INLINE std::basic_ostream& 36 | operator<<(std::basic_ostream& out, variant const& rhs) 37 | { 38 | detail::printer> visitor(out); 39 | apply_visitor(visitor, rhs); 40 | return out; 41 | } 42 | } // namespace util 43 | } // namespace mapbox 44 | 45 | #endif // MAPBOX_UTIL_VARIANT_IO_HPP 46 | -------------------------------------------------------------------------------- /contrib/variant/1.2.0/include/mapbox/variant_visitor.hpp: -------------------------------------------------------------------------------- 1 | #ifndef MAPBOX_UTIL_VARIANT_VISITOR_HPP 2 | #define MAPBOX_UTIL_VARIANT_VISITOR_HPP 3 | 4 | #include 5 | 6 | namespace mapbox { 7 | namespace util { 8 | 9 | template 10 | struct visitor; 11 | 12 | template 13 | struct visitor : Fn 14 | { 15 | using Fn::operator(); 16 | 17 | template 18 | visitor(T&& fn) : Fn(std::forward(fn)) {} 19 | }; 20 | 21 | template 22 | struct visitor : Fn, visitor 23 | { 24 | using Fn::operator(); 25 | using visitor::operator(); 26 | 27 | template 28 | visitor(T&& fn, Ts&&... fns) 29 | : Fn(std::forward(fn)) 30 | , visitor(std::forward(fns)...) {} 31 | }; 32 | 33 | template 34 | visitor::type...> make_visitor(Fns&&... fns) 35 | { 36 | return visitor::type...> 37 | (std::forward(fns)...); 38 | } 39 | 40 | } // namespace util 41 | } // namespace mapbox 42 | 43 | #endif // MAPBOX_UTIL_VARIANT_VISITOR_HPP 44 | -------------------------------------------------------------------------------- /contrib/vtzero/1.1.0/LICENSE: -------------------------------------------------------------------------------- 1 | BSD 2-Clause License 2 | 3 | Copyright (c) 2017, Mapbox 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 20 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 22 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 23 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | -------------------------------------------------------------------------------- /contrib/vtzero/1.1.0/include/vtzero/output.hpp: -------------------------------------------------------------------------------- 1 | #ifndef VTZERO_OUTPUT_HPP 2 | #define VTZERO_OUTPUT_HPP 3 | 4 | /***************************************************************************** 5 | 6 | vtzero - Tiny and fast vector tile decoder and encoder in C++. 7 | 8 | This file is from https://github.com/mapbox/vtzero where you can find more 9 | documentation. 10 | 11 | *****************************************************************************/ 12 | 13 | /** 14 | * @file output.hpp 15 | * 16 | * @brief Contains overloads of operator<< for basic vtzero types. 17 | */ 18 | 19 | #include 20 | #include 21 | 22 | #include 23 | 24 | namespace vtzero { 25 | 26 | /// Overload of the << operator for GeomType 27 | template 28 | std::basic_ostream& operator<<(std::basic_ostream& out, const GeomType type) { 29 | return out << geom_type_name(type); 30 | } 31 | 32 | /// Overload of the << operator for property_value_type 33 | template 34 | std::basic_ostream& operator<<(std::basic_ostream& out, const property_value_type type) { 35 | return out << property_value_type_name(type); 36 | } 37 | 38 | /// Overload of the << operator for index_value 39 | template 40 | std::basic_ostream& operator<<(std::basic_ostream& out, const index_value index) { 41 | if (index.valid()) { 42 | return out << index.value(); 43 | } 44 | return out << "invalid"; 45 | } 46 | 47 | /// Overload of the << operator for index_value_pair 48 | template 49 | std::basic_ostream& operator<<(std::basic_ostream& out, const index_value_pair index_pair) { 50 | if (index_pair.valid()) { 51 | return out << '[' << index_pair.key() << ',' << index_pair.value() << ']'; 52 | } 53 | return out << "invalid"; 54 | } 55 | 56 | /// Overload of the << operator for points 57 | template 58 | std::basic_ostream& operator<<(std::basic_ostream& out, const point p) { 59 | return out << '(' << p.x << ',' << p.y << ')'; 60 | } 61 | 62 | } // namespace vtzero 63 | 64 | #endif // VTZERO_OUTPUT_HPP 65 | -------------------------------------------------------------------------------- /contrib/vtzero/1.1.0/include/vtzero/property.hpp: -------------------------------------------------------------------------------- 1 | #ifndef VTZERO_PROPERTY_HPP 2 | #define VTZERO_PROPERTY_HPP 3 | 4 | /***************************************************************************** 5 | 6 | vtzero - Tiny and fast vector tile decoder and encoder in C++. 7 | 8 | This file is from https://github.com/mapbox/vtzero where you can find more 9 | documentation. 10 | 11 | *****************************************************************************/ 12 | 13 | /** 14 | * @file property.hpp 15 | * 16 | * @brief Contains the property class. 17 | */ 18 | 19 | #include "property_value.hpp" 20 | #include "types.hpp" 21 | 22 | namespace vtzero { 23 | 24 | /** 25 | * A view of a vector tile property (key and value). 26 | * 27 | * Doesn't hold any data itself, just views of the key and value. 28 | */ 29 | class property { 30 | 31 | data_view m_key{}; 32 | property_value m_value{}; 33 | 34 | public: 35 | 36 | /** 37 | * The default constructor creates an invalid (empty) property. 38 | */ 39 | constexpr property() noexcept = default; 40 | 41 | /** 42 | * Create a (valid) property from a key and value. 43 | */ 44 | constexpr property(const data_view key, const property_value value) noexcept : 45 | m_key(key), 46 | m_value(value) { 47 | } 48 | 49 | /** 50 | * Is this a valid property? Properties are valid if they were 51 | * constructed using the non-default constructor. 52 | */ 53 | constexpr bool valid() const noexcept { 54 | return m_key.data() != nullptr; 55 | } 56 | 57 | /** 58 | * Is this a valid property? Properties are valid if they were 59 | * constructed using the non-default constructor. 60 | */ 61 | explicit constexpr operator bool() const noexcept { 62 | return valid(); 63 | } 64 | 65 | /// Return the property key. 66 | constexpr data_view key() const noexcept { 67 | return m_key; 68 | } 69 | 70 | /// Return the property value. 71 | constexpr property_value value() const noexcept { 72 | return m_value; 73 | } 74 | 75 | }; // class property 76 | 77 | /// properties are equal if they contain the same key & value data. 78 | inline constexpr bool operator==(const property& lhs, const property& rhs) noexcept { 79 | return lhs.key() == rhs.key() && lhs.value() == rhs.value(); 80 | } 81 | 82 | /// properties are unequal if they do not contain them same key and value data. 83 | inline constexpr bool operator!=(const property& lhs, const property& rhs) noexcept { 84 | return !(lhs == rhs); 85 | } 86 | 87 | } // namespace vtzero 88 | 89 | #endif // VTZERO_PROPERTY_HPP 90 | -------------------------------------------------------------------------------- /contrib/vtzero/1.1.0/include/vtzero/property_mapper.hpp: -------------------------------------------------------------------------------- 1 | #ifndef VTZERO_PROPERTY_MAPPER_HPP 2 | #define VTZERO_PROPERTY_MAPPER_HPP 3 | 4 | /***************************************************************************** 5 | 6 | vtzero - Tiny and fast vector tile decoder and encoder in C++. 7 | 8 | This file is from https://github.com/mapbox/vtzero where you can find more 9 | documentation. 10 | 11 | *****************************************************************************/ 12 | 13 | /** 14 | * @file property_mapper.hpp 15 | * 16 | * @brief Contains the property_mapper class. 17 | */ 18 | 19 | #include "builder.hpp" 20 | #include "layer.hpp" 21 | 22 | #include 23 | 24 | namespace vtzero { 25 | 26 | /** 27 | * Establishes a mapping between index values of properties of an existing 28 | * layer and a new layer. Can be used when copying some features from an 29 | * existing layer to a new layer. 30 | */ 31 | class property_mapper { 32 | 33 | const layer& m_layer; 34 | layer_builder& m_layer_builder; 35 | 36 | std::vector m_keys; 37 | std::vector m_values; 38 | 39 | public: 40 | 41 | /** 42 | * Construct the mapping between the specified layers 43 | * 44 | * @param layer The existing layer from which (some) properties will 45 | * be copied. 46 | * @param layer_builder The new layer that is being created. 47 | */ 48 | property_mapper(const layer& layer, layer_builder& layer_builder) : 49 | m_layer(layer), 50 | m_layer_builder(layer_builder) { 51 | m_keys.resize(layer.key_table().size()); 52 | m_values.resize(layer.value_table().size()); 53 | } 54 | 55 | /** 56 | * Map the value index of a key. 57 | * 58 | * @param index The value index of the key in the existing table. 59 | * @returns The value index of the same key in the new table. 60 | */ 61 | index_value map_key(index_value index) { 62 | auto& k = m_keys[index.value()]; 63 | 64 | if (!k.valid()) { 65 | k = m_layer_builder.add_key_without_dup_check(m_layer.key(index)); 66 | } 67 | 68 | return k; 69 | } 70 | 71 | /** 72 | * Map the value index of a value. 73 | * 74 | * @param index The value index of the value in the existing table. 75 | * @returns The value index of the same value in the new table. 76 | */ 77 | index_value map_value(index_value index) { 78 | auto& v = m_values[index.value()]; 79 | 80 | if (!v.valid()) { 81 | v = m_layer_builder.add_value_without_dup_check(m_layer.value(index)); 82 | } 83 | 84 | return v; 85 | } 86 | 87 | /** 88 | * Map the value indexes of a key/value pair. 89 | * 90 | * @param idxs The value indexes of the key/value pair in the existing 91 | * table. 92 | * @returns The value indexes of the same key/value pair in the new 93 | * table. 94 | */ 95 | index_value_pair operator()(index_value_pair idxs) { 96 | return {map_key(idxs.key()), map_value(idxs.value())}; 97 | } 98 | 99 | }; // class property_mapper 100 | 101 | } // namespace vtzero 102 | 103 | #endif // VTZERO_PROPERTY_MAPPER_HPP 104 | -------------------------------------------------------------------------------- /contrib/vtzero/1.1.0/include/vtzero/version.hpp: -------------------------------------------------------------------------------- 1 | #ifndef VTZERO_VERSION_HPP 2 | #define VTZERO_VERSION_HPP 3 | 4 | /***************************************************************************** 5 | 6 | vtzero - Tiny and fast vector tile decoder and encoder in C++. 7 | 8 | This file is from https://github.com/mapbox/vtzero where you can find more 9 | documentation. 10 | 11 | *****************************************************************************/ 12 | 13 | /** 14 | * @file version.hpp 15 | * 16 | * @brief Contains the version number macros for the vtzero library. 17 | */ 18 | 19 | /// The major version number 20 | #define VTZERO_VERSION_MAJOR 1 21 | 22 | /// The minor version number 23 | #define VTZERO_VERSION_MINOR 1 24 | 25 | /// The patch number 26 | #define VTZERO_VERSION_PATCH 0 27 | 28 | /// The complete version number 29 | #define VTZERO_VERSION_CODE \ 30 | (VTZERO_VERSION_MAJOR * 10000 + VTZERO_VERSION_MINOR * 100 + \ 31 | VTZERO_VERSION_PATCH) 32 | 33 | /// Version number as string 34 | #define VTZERO_VERSION_STRING "1.1.0" 35 | 36 | #endif // VTZERO_VERSION_HPP 37 | -------------------------------------------------------------------------------- /docs/Add-New-Rule.md: -------------------------------------------------------------------------------- 1 | # Add a basic rule 2 | To add a new rule, a YAML Rule file should be created inside the `analyser/rules_specifications` folder. The YAML file name should be unique and it will be equal to the name of the rule (as much as possible). 3 | 4 | The content of the file should follows the [YAML Rule Specification](YAML-Rule-Specification.md) format. 5 | 6 | # Create custom pipes (python code) 7 | 8 | If a more advanced rule needs a custom python logic, a custom `Pipe` can be created. 9 | To do that, a folder with the name of the rule should be created inside the `analyser/core/pipes/rules_specific_pipes` folder. 10 | 11 | A python file should be created inside the newly created folder. This file should contain a class which inherits from the `Pipe` base class. See the [Pipe base class](Pipes.md#The-Pipe-base-class) chapter to understand how the base `Pipe` class works and how to create a custom pipe from it. 12 | 13 | The custom `Pipe` class should be imported inside the `__init__.py` of the `analyser/core/pipes/rules_specific_pipes` so that it can be found by the `Assembler` (See [Architecture](Overview.md#architecture)). 14 | 15 | Once the custom pipe well created, it can be used in the [YAML Rule Specification](YAML-Rule-Specification.md) by setting its `class` name as the value of the `type` field. -------------------------------------------------------------------------------- /docs/Dynamic-Value.md: -------------------------------------------------------------------------------- 1 | # The DynamicValue type 2 | 3 | ## DynamicValue class 4 | 5 | A DynamicValue is an object which should be resolved at runtime to return a concrete value. 6 | 7 | Data extracted by the query in a rule can have different fields values which should be handled differently by the pipes. The DynamicValue class was introduced to allow this while keeping a static [YAML Rule Specification](YAML-Rule-Specification.md). 8 | 9 | There is multiple type of dynamic value but each of them inherits from the DynamicValue base class. 10 | 11 | This class contains one important abstract method which is implemented by each subclasses, the `resolve` method: 12 | 13 | ```python 14 | @abstractmethod 15 | def resolve(self, data: Dict) -> Any: 16 | """ 17 | Assigns a concrete value to the dynamic value 18 | based on the input data dictionnary. 19 | """ 20 | return 21 | ``` 22 | 23 | The input data dictionnary are the data which will be used to resolve the dynamic value. 24 | 25 | We can take the [Variable](../analyser/core/dynamic_value/variable.py) class as an example. The resolve method basically searchs for the `name` key in the input data dictionnary where `name` is the variable name given in the YAML Specification. The value corresponding to this key is then returned. 26 | 27 | ## Resolver 28 | 29 | The `resolver` is used to easily resolve DynamicValue classes even when there are nested DynamicValue: 30 | 31 | ```yaml 32 | - !switch 33 | expression: osm_type 34 | cases: 35 | 'N': 36 | node_id: !variable osm_id 37 | 'W': 38 | way_id: !variable osm_id 39 | 'R': 40 | relation_id: !variable osm_id 41 | ``` 42 | 43 | In this specification we can see that there is `!variable` types nested inside a `!switch` type. When the switch will be resolved, a `Variable` object will be returned, hence it needs to also be resolved. 44 | 45 | The [resolver](../analyser/core/dynamic_value/resolver.py) can handle this easily when we call the `resolve_all` function: 46 | 47 | ```python 48 | def resolve_all(data_to_resolve: Any, resolver_data: Dict, ) -> Any: 49 | """ 50 | Resolves the given data_to_resolve by resolving all data inside which are of type DynamicValue. The resolved data are resolved again if they also contain DynamicValue. 51 | 52 | Parameter resolver_data is the data dictionnary used to resolve the dynamic values. 53 | """ 54 | ``` 55 | 56 | This function is for example used in the `process` method of the [GeoJSONFeatureConverter](../analyser/core/pipes/output_formatters/geojson_feature_converter.py) to resolve the properties dynamically. This is needed because often we want to create different properties based on the geometry type of the GeoJSON feature (node, way, relation). 57 | 58 | ## All dynamic values 59 | 60 | The dynamic values currently implemented are the following: 61 | 62 | * Switch (!switch) 63 | * Variable (!variable) 64 | 65 | More explanation on them and especially on their YAML Specification can be found in the [Syntax section (under custom types) of the YAML Rule Specification chapter](YAML-Rule-Specification.md#Syntax). -------------------------------------------------------------------------------- /docs/Overview.md: -------------------------------------------------------------------------------- 1 | # Introduction 2 | 3 | The Nominatim Data Analyser is a QA tool used to scan the nominatim database and extract 4 | suspect data from it. 5 | 6 | Once extracted the data are processed and converted into some output formats like GeoJSON, vector tiles, etc. 7 | The purpose is to use these outputs into other tools like [osmoscope-ui](https://github.com/osmoscope/osmoscope-ui). 8 | 9 | When displayed through a public visualization tool, the data can be corrected by mappers directly. 10 | 11 | The whole purpose of the Nominatim Data Analyser is to increase the quality of OpenStreetMap data and therefore indirectly the search results of Nominatim. 12 | 13 | # Rules 14 | 15 | This tool works by executing a set of `rules` where a `rule` is a definition 16 | of what should be done from the data extraction to the outputs generation. 17 | 18 | Each rule is defined inside a YAML file following the [YAML Rule Specification](YAML-Rule-Specification.md). This system has been designed in order to provide an easy and visual way of creating a new rule. 19 | 20 | All rules can be found in the `analyser/rules_specifications` folder. 21 | 22 | To add a new rule follow the [Add a new rule chapter](Add-New-Rule.md). 23 | 24 | # Architecture 25 | 26 | The tool is designed around a global `Pipeline` architecture where each data processing unit is defined as a class which inherits from the `Pipe` base class. One rule is equal to one pipeline constructed from the YAML Rule file. 27 | 28 | The pipeline of a rule is assembled by the `Assembler` module. The `Assembler` module assembles pipe in the right order by receiving data from the `Deconstructor` module to which it is subscribed. 29 | 30 | The `Deconstructor` module gets a pipeline specification as input (for a rule it gets loaded from the YAML Rule file). A pipeline specification follows a tree structure where each node is equal to a `Pipe`. The `Deconstructor` explores this tree structure and sends data through events throughout the deconstruction process. 31 | 32 | When the `Deconstructor` encounters a new node it sends the node's data to its subscribers. Thus, the `Assembler` knows that it should assemble a new `Pipe` following the data it just received. The `Assembler` knows that it should plug this newly assembled `Pipe` to the last `Pipe` assembled because it receives pipes in the right order (thanks to the tree structure and the `Deconstructor`). 33 | 34 | When a leaf is reached by the `Deconstructor`, it backtracks through the tree until it finds an unexplored path or until it is done. The `Deconstructor` notifies it subscribers when backtracking so that the `Assembler` can keep track of the pipes order. 35 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools>=42", "pybind11>=2.10.0"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [project] 6 | name = "nominatim-data-analyser" 7 | authors = [ 8 | {name = "Antonin Jolivat", email = "antonin.jolivat@gmail.com"} 9 | ] 10 | maintainers = [ 11 | {name = "Sarah Hoffmann", email = "lonvia@denofr.de"} 12 | ] 13 | description = "QA Tool for Nominatim. Helps to improve the OpenStreetMap data quality and therefore the Nominatim search results." 14 | readme = "README.md" 15 | requires-python = ">=3.10" 16 | license = {text = "GPL-2.0-or-later"} 17 | classifiers = [ 18 | "Programming Language :: Python :: 3", 19 | ] 20 | dependencies = [ 21 | "pyYAML", 22 | "geojson", 23 | "psycopg" 24 | ] 25 | version = "0.1.0" 26 | 27 | [project.urls] 28 | Homepage = "https://github.com/osm-search/Nominatim-Data-Analyser" 29 | 30 | [project.optional-dependencies] 31 | tests = [ 32 | 'pytest', 33 | ] 34 | 35 | [tool.setuptools] 36 | packages = ["nominatim_data_analyser", 37 | "nominatim_data_analyser.logger", 38 | "nominatim_data_analyser.core", 39 | "nominatim_data_analyser.core.exceptions", 40 | "nominatim_data_analyser.core.yaml_logic", 41 | "nominatim_data_analyser.core.dynamic_value", 42 | "nominatim_data_analyser.core.deconstructor", 43 | "nominatim_data_analyser.core.model", 44 | "nominatim_data_analyser.core.pipes", 45 | "nominatim_data_analyser.core.pipes.rules_specific_pipes", 46 | "nominatim_data_analyser.core.pipes.rules_specific_pipes.place_nodes_close", 47 | "nominatim_data_analyser.core.pipes.rules_specific_pipes.addr_house_number_no_digit", 48 | "nominatim_data_analyser.core.pipes.rules_specific_pipes.duplicate_label_role", 49 | "nominatim_data_analyser.core.pipes.rules_specific_pipes.same_wikidata", 50 | "nominatim_data_analyser.core.pipes.output_formatters", 51 | "nominatim_data_analyser.core.pipes.data_processing", 52 | "nominatim_data_analyser.core.pipes.data_fetching", 53 | "nominatim_data_analyser.core.qa_rule", 54 | "nominatim_data_analyser.core.assembler", 55 | "nominatim_data_analyser.rules_specifications" 56 | ] 57 | 58 | package-dir = {"" = "src"} 59 | 60 | [project.scripts] 61 | nominatim-data-analyser = "nominatim_data_analyser.cli:cli" 62 | 63 | [tool.pytest.ini_options] 64 | testpaths = ["tests"] 65 | 66 | [tool.mypy] 67 | strict = true 68 | python_version = "3.10" 69 | files = "src" 70 | 71 | [[tool.mypy.overrides]] 72 | module = ["geojson", "geojson.feature", "geojson.geometry"] 73 | ignore_missing_imports = true 74 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | from pybind11.setup_helpers import Pybind11Extension 3 | 4 | ext_modules = [ 5 | Pybind11Extension( 6 | "nominatim_data_analyser.clustering_vt", 7 | ["clustering-vt/clustering-vt.cpp"], 8 | include_dirs=[ 9 | "contrib/geojson/0.4.3/include", 10 | "contrib/protozero/1.7.0/include", 11 | "contrib/geometry/1.0.0/include", 12 | "contrib/vtzero/1.1.0/include", 13 | "contrib/kdbush/0.1.3/include", 14 | "contrib/supercluster/0.3.2/include", 15 | "contrib/variant/1.2.0/include", 16 | "contrib/rapidjson/1.1.0/include", 17 | ], 18 | cxx_std=17 19 | ), 20 | ] 21 | 22 | setup(ext_modules=ext_modules) 23 | -------------------------------------------------------------------------------- /src/nominatim_data_analyser/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/osm-search/Nominatim-Data-Analyser/57cd6060ab506e3f7bf0b0e50f1fd1aa25e91567/src/nominatim_data_analyser/__init__.py -------------------------------------------------------------------------------- /src/nominatim_data_analyser/cli.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | from .core.core import Core 4 | 5 | def cli() -> int: 6 | parser = argparse.ArgumentParser(prog='nominatim-analyser') 7 | 8 | parser.add_argument('--execute-all', action='store_true', 9 | help='Executes all the QA rules') 10 | parser.add_argument('--filter', metavar='', nargs='+', 11 | help='Filters some QA rules so they are not executed.') 12 | parser.add_argument('--execute-one', metavar='', action='store', 13 | type=str, help='Executes the given QA rule') 14 | parser.add_argument('--config', metavar='', default='config.yaml', 15 | help='Location of config file (default: config.yaml)') 16 | 17 | args = parser.parse_args() 18 | 19 | core = Core(config_file=args.config) 20 | 21 | # Executes all the QA rules. If a filter is given, these rules are excluded from the execution. 22 | if args.execute_all: 23 | core.execute_all(args.filter) 24 | elif args.execute_one: 25 | # Execute the given QA rule. 26 | core.execute_one(args.execute_one) 27 | else: 28 | return 1 29 | 30 | return 0 31 | -------------------------------------------------------------------------------- /src/nominatim_data_analyser/config.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import yaml 4 | 5 | from .logger.logger import LOG 6 | 7 | 8 | class Config: 9 | values: dict[str, str] = {} 10 | 11 | 12 | def load_config(config_file: Path | None) -> None: 13 | """ 14 | Load the YAML config file into the 15 | config global variable. 16 | """ 17 | Config.values.clear() 18 | 19 | # First load the default settings. 20 | _get_config_file_contents(Path(__file__, '..', 'default_config.yaml').resolve()) 21 | 22 | # Then overwrite with potential custom settings. 23 | if config_file is not None and config_file.is_file(): 24 | LOG.info(f"Loading config from {config_file}.") 25 | _get_config_file_contents(config_file) 26 | 27 | 28 | def _get_config_file_contents(config_file: Path) -> None: 29 | with config_file.open('r') as file: 30 | try: 31 | contents = yaml.safe_load(file) 32 | except yaml.YAMLError as exc: 33 | LOG.error(f"Error while loading the config file: {exc}") 34 | raise 35 | 36 | if not isinstance(contents, dict): 37 | raise RuntimeError('Error in config file, expected key-value entries.') 38 | 39 | for k, v in contents.items(): 40 | if not isinstance(k, str): 41 | raise RuntimeError(f"Error in config file, non-string key {k}.") 42 | Config.values[k] = str(v) 43 | -------------------------------------------------------------------------------- /src/nominatim_data_analyser/core/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Core module of the analyser 3 | """ 4 | from .pipe import Pipe as Pipe 5 | -------------------------------------------------------------------------------- /src/nominatim_data_analyser/core/assembler/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module handling the assembly of a rule's pipeline. 3 | """ 4 | 5 | from .pipeline_assembler import PipelineAssembler as PipelineAssembler 6 | from .pipe_factory import PipeFactory as PipeFactory 7 | -------------------------------------------------------------------------------- /src/nominatim_data_analyser/core/assembler/pipe_factory.py: -------------------------------------------------------------------------------- 1 | from typing import Any, cast 2 | from ..exceptions import YAMLSyntaxException 3 | from .. import pipes as pipes_module 4 | from .. import Pipe 5 | from ..qa_rule import ExecutionContext 6 | 7 | class PipeFactory(): 8 | """ 9 | Factory to assemble pipes. 10 | """ 11 | @staticmethod 12 | def assemble_pipe(node_data: dict[str, Any], exec_context: ExecutionContext) -> Pipe: 13 | """ 14 | Instantiate a pipe based on the given node_data 15 | """ 16 | if 'type' not in node_data: 17 | raise YAMLSyntaxException("Each node of the tree (pipe) should have a type defined.") 18 | 19 | try: 20 | assembled_pipe = cast(Pipe, getattr(pipes_module, node_data['type'])(node_data, exec_context)) 21 | except AttributeError: 22 | raise YAMLSyntaxException(f"The type {node_data['type']} doesn't exist.") 23 | 24 | return assembled_pipe 25 | -------------------------------------------------------------------------------- /src/nominatim_data_analyser/core/assembler/pipeline_assembler.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | from ..deconstructor import PipelineDeconstructor, BACKTRACKING_EVENT, NEW_NODE_EVENT 3 | from .. import Pipe 4 | from ..pipes import FillingPipe 5 | from ..qa_rule import ExecutionContext 6 | from ...logger.logger import LOG 7 | from collections import deque 8 | from .pipe_factory import PipeFactory 9 | 10 | 11 | class PipelineAssembler(): 12 | """ 13 | Get deconstruction informations from the 14 | pipeline deconstructor and assembles the final pipeline. 15 | """ 16 | def __init__(self, pipeline_specification: dict[str, Any], rule_name: str) -> None: 17 | self.rule_name = rule_name 18 | self.deconstructor: PipelineDeconstructor = PipelineDeconstructor(pipeline_specification, rule_name) 19 | self.deconstructor.subscribe_event(NEW_NODE_EVENT, self.on_new_node) 20 | self.deconstructor.subscribe_event(BACKTRACKING_EVENT, self.on_backtrack) 21 | self.nodes_history: deque[Pipe] = deque() 22 | self.exec_context: ExecutionContext = ExecutionContext() 23 | self.exec_context.rule_name = rule_name 24 | self.first_pipe: Pipe = FillingPipe({}, self.exec_context) 25 | 26 | def on_new_node(self, node: dict[str, Any]) -> None: 27 | """ 28 | Raised by the deconstructor when a new node 29 | is reached. 30 | """ 31 | if node['type'] == 'ROOT_NODE': 32 | self.nodes_history.append(self.first_pipe) 33 | else: 34 | pipe = PipeFactory.assemble_pipe(node, self.exec_context) 35 | # Plug the new pipe to the current last pipe of the deque 36 | self.nodes_history[-1].plug_pipe(pipe) 37 | LOG.info("<%s> Assembler -> %s plugged to %s", self.rule_name, pipe, self.nodes_history[-1]) 38 | self.nodes_history.append(pipe) 39 | 40 | def on_backtrack(self) -> None: 41 | """ 42 | Raised by the deconstructor when backtrack is 43 | processed through the tree. 44 | """ 45 | if self.nodes_history: 46 | self.nodes_history.pop() 47 | 48 | def assemble(self) -> Pipe: 49 | """ 50 | Assembles the full pipeline. 51 | """ 52 | self.deconstructor.deconstruct() 53 | return self.first_pipe 54 | -------------------------------------------------------------------------------- /src/nominatim_data_analyser/core/core.py: -------------------------------------------------------------------------------- 1 | from .yaml_logic.yaml_loader import load_yaml_rule 2 | from .assembler.pipeline_assembler import PipelineAssembler 3 | from ..logger.logger import LOG 4 | from ..logger.timer import Timer 5 | from ..config import load_config 6 | from pathlib import Path 7 | 8 | class Core(): 9 | """ 10 | Core of the analyser used to execute rules. 11 | """ 12 | def __init__(self, config_file: str | Path | None) -> None: 13 | load_config(None if config_file is None else Path(config_file)) 14 | self.rules_path = Path(__file__, '..', '..', 'rules_specifications').resolve() 15 | 16 | def execute_all(self, filter: list[str] | None = None) -> None: 17 | """ 18 | Execute each QA rules. 19 | 20 | If a filter is given as parameter, the rules inside this 21 | filter wont be executed. 22 | """ 23 | for rule_file in self.rules_path.glob('*.yaml'): 24 | if not filter or rule_file.stem not in filter: 25 | self._execute(rule_file) 26 | 27 | def execute_one(self, name: str) -> None: 28 | """ 29 | Execute one QA rule based on its YAML file name. 30 | """ 31 | self._execute(self.rules_path / f"{name}.yaml") 32 | 33 | def _execute(self, rule_file: Path) -> None: 34 | timer = Timer().start_timer() 35 | loaded_yaml = load_yaml_rule(rule_file) 36 | PipelineAssembler(loaded_yaml, rule_file.stem).assemble().process_and_next() 37 | LOG.info('<%s> The whole rule executed in %s mins %s secs', rule_file.stem, *timer.get_elapsed()) 38 | -------------------------------------------------------------------------------- /src/nominatim_data_analyser/core/deconstructor/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module handling the deconstruction of 3 | YAML rules specifications to build their 4 | corresponding pipeline. 5 | """ 6 | 7 | from .pipeline_deconstructor import * -------------------------------------------------------------------------------- /src/nominatim_data_analyser/core/dynamic_value/__init__.py: -------------------------------------------------------------------------------- 1 | from .dynamic_value import DynamicValue as DynamicValue 2 | from .switch import Switch as Switch 3 | from .variable import Variable as Variable 4 | from .resolver import (resolve_all as resolve_all, 5 | resolve_one as resolve_one) 6 | -------------------------------------------------------------------------------- /src/nominatim_data_analyser/core/dynamic_value/dynamic_value.py: -------------------------------------------------------------------------------- 1 | from abc import ABCMeta, abstractmethod 2 | from typing import Any 3 | 4 | class DynamicValue(metaclass=ABCMeta): 5 | """ 6 | Base class for every DynamicValue type. 7 | 8 | A DynamicValue is meant to be created from a special type in the 9 | YAML Specification and it allows to replace some fields 10 | dynamically depending on the data values. 11 | """ 12 | @abstractmethod 13 | def resolve(self, data: dict[str, Any]) -> Any: 14 | """ 15 | Assigns a concrete value to the dynamic value 16 | based on the input data dictionnary. 17 | """ 18 | return 19 | -------------------------------------------------------------------------------- /src/nominatim_data_analyser/core/dynamic_value/resolver.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Dict, Iterable 2 | 3 | from . import DynamicValue 4 | 5 | def resolve_all(data_to_resolve: Any, resolver_data: dict[str, Any]) -> Any: 6 | """ 7 | Resolves the given data_to_resolve by resolving all data inside which are 8 | of type DynamicValue. The resolved data are resolved again if they also contain 9 | DynamicValue. 10 | 11 | Parameter resolver_data is the data dictionnary used to resolve the dynamic values. 12 | """ 13 | while is_resolvable(data_to_resolve): 14 | data_to_resolve = resolve_one(data_to_resolve, resolver_data) 15 | return data_to_resolve 16 | 17 | def resolve_one(data_to_resolve: Any, resolver_data: dict[str, Any]) -> Any: 18 | """ 19 | Resolves every DynamicValue of the given data_to_resolve. 20 | If the given data is an Iterable all values of type DynamicValue 21 | are resolved. If the given data is a Dictionnary every keys and values 22 | are resolved if of type DynamicValue. 23 | """ 24 | if isinstance(data_to_resolve, Iterable): 25 | # Resolve dictionnary by trying to resolve each key and each value. 26 | if isinstance(data_to_resolve, Dict): 27 | new_dict = dict() 28 | for k, v in data_to_resolve.items(): 29 | k = _resolve_if_resolvable(resolver_data, k) 30 | v = _resolve_if_resolvable(resolver_data, v) 31 | new_dict[k] = v 32 | data_to_resolve = new_dict 33 | # Resolve all others classic iterables. 34 | else: 35 | data_to_resolve = type(data_to_resolve)( 36 | map(lambda x: _resolve_if_resolvable(resolver_data, x), data_to_resolve)) # type: ignore[call-arg] 37 | else: 38 | data_to_resolve = _resolve_if_resolvable(resolver_data, data_to_resolve) 39 | 40 | return data_to_resolve 41 | 42 | def is_resolvable(data: Any) -> bool: 43 | """ 44 | Checks if the given data contains any DynamicValue which 45 | is resolvable. If the data is an Iterable every value are checked. 46 | Also, if the data is a Dictionnary every keys and every values are checked. 47 | """ 48 | if isinstance(data, Iterable): 49 | if isinstance(data, dict): 50 | # Checks all the keys and values of the dictionnary. 51 | return _contains_dynamic_value(data.keys()) \ 52 | or _contains_dynamic_value(data.values()) 53 | else: 54 | return _contains_dynamic_value(data) 55 | else: 56 | return _is_dynamic_value(data) 57 | 58 | def _contains_dynamic_value(data: Iterable[Any]) -> bool: 59 | """ 60 | Checks if the given Iterable contains any instance 61 | of the DynamicValue type. 62 | """ 63 | return any([_is_dynamic_value(x) for x in data]) 64 | 65 | def _is_dynamic_value(data: Any) -> bool: 66 | """ 67 | Checks if the given value is an instance of the DynamicValue type. 68 | """ 69 | return isinstance(data, DynamicValue) 70 | 71 | def _resolve_if_resolvable(resolver_data: dict[str, Any], data: Any) -> Any: 72 | """ 73 | If the given data is of type DynamicValue it gets resolved. 74 | Otherwise it gets returned as it is. 75 | """ 76 | return data.resolve(resolver_data) if _is_dynamic_value(data) else data 77 | -------------------------------------------------------------------------------- /src/nominatim_data_analyser/core/dynamic_value/switch.py: -------------------------------------------------------------------------------- 1 | from . import DynamicValue 2 | from typing import Any 3 | 4 | class Switch(DynamicValue): 5 | """ 6 | Dynamic value implementing a switch condition. 7 | The value of the expression is evaluated against the 8 | cases to get the right value. 9 | """ 10 | def __init__(self, expression: str, cases: dict[str, Any]) -> None: 11 | self.expression = expression 12 | self.cases = cases 13 | 14 | def resolve(self, data: dict[str, Any]) -> Any: 15 | if self.expression not in data: 16 | raise Exception(f'The expression {self.expression} was not found in the input dictionnary.') 17 | 18 | if data[self.expression] not in self.cases: 19 | raise Exception(f'The case {data[self.expression]} is not in ' 20 | f'the configured switch cases: {list(self.cases.keys())}') 21 | 22 | return self.cases[data[self.expression]] 23 | -------------------------------------------------------------------------------- /src/nominatim_data_analyser/core/dynamic_value/variable.py: -------------------------------------------------------------------------------- 1 | from . import DynamicValue 2 | from typing import Any 3 | 4 | class Variable(DynamicValue): 5 | """ 6 | Dynamic value corresponding to a basic variable. 7 | The right value is extracted based on its key name. 8 | """ 9 | def __init__(self, name: str) -> None: 10 | self.name = name 11 | 12 | def resolve(self, data: dict[str, Any]) -> Any: 13 | if self.name not in data: 14 | raise Exception(f'The variable name {self.name} was not found in the input dictionary.') 15 | 16 | return data[self.name] 17 | -------------------------------------------------------------------------------- /src/nominatim_data_analyser/core/exceptions/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Contains custom exceptions 3 | """ 4 | 5 | from .yaml_syntax_exception import YAMLSyntaxException as YAMLSyntaxException 6 | -------------------------------------------------------------------------------- /src/nominatim_data_analyser/core/exceptions/yaml_syntax_exception.py: -------------------------------------------------------------------------------- 1 | class YAMLSyntaxException(Exception): 2 | """ 3 | Custom exception for syntax errors in YAML. 4 | """ 5 | pass 6 | -------------------------------------------------------------------------------- /src/nominatim_data_analyser/core/model/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module containing the generic data schema 3 | used by the analyzer to convey data independently 4 | to any module of the pipeline. 5 | """ 6 | 7 | from .node import Node 8 | from .geometry import Geometry -------------------------------------------------------------------------------- /src/nominatim_data_analyser/core/model/geometry.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | from abc import ABCMeta, abstractmethod 3 | from dataclasses import dataclass 4 | from geojson.feature import Feature 5 | 6 | @dataclass 7 | class Geometry(metaclass=ABCMeta): 8 | @abstractmethod 9 | def to_geojson_feature(self, id: int, properties: dict[str, str]) -> Feature: ... 10 | 11 | @staticmethod 12 | @abstractmethod 13 | def create_from_WKT_string(geom: str) -> Geometry: 14 | """ 15 | Parse a WKT geometry to instantiate a new 16 | Geometry class. 17 | """ 18 | -------------------------------------------------------------------------------- /src/nominatim_data_analyser/core/model/node.py: -------------------------------------------------------------------------------- 1 | from geojson.feature import Feature 2 | from geojson.geometry import Point 3 | from .geometry import Geometry 4 | from dataclasses import dataclass 5 | import re 6 | 7 | @dataclass 8 | class Node(Geometry): 9 | coordinates: list[float] 10 | 11 | def to_geojson_feature(self, id: int, properties: dict[str, str]) -> Feature: 12 | """ 13 | Return this Node as a geojson feature. 14 | """ 15 | point = Point(coordinates=self.coordinates) 16 | feature = Feature(geometry=point, properties=properties, id=id) 17 | return feature 18 | 19 | @staticmethod 20 | def create_from_WKT_string(geom: str) -> 'Node': 21 | """ 22 | Parse a WKT geometry to instantiate a new 23 | Node class. 24 | """ 25 | pattern = r"POINT\((.*) (.*)\)" 26 | results = re.match(pattern, geom) 27 | if results is None: 28 | raise RuntimeError("Not a valid WKT.") 29 | coordinates = [float(results[1]), float(results[2])] 30 | return Node(coordinates) 31 | -------------------------------------------------------------------------------- /src/nominatim_data_analyser/core/pipe.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | import uuid 3 | import logging 4 | from abc import ABCMeta, abstractmethod 5 | from ..logger.logger import LOG 6 | from .exceptions import YAMLSyntaxException 7 | from .qa_rule import ExecutionContext 8 | 9 | 10 | class Pipe(metaclass=ABCMeta): 11 | """ 12 | This is the base class for every pipe. 13 | """ 14 | def __init__(self, data: dict[str, Any], exec_context: ExecutionContext) -> None: 15 | self.id = uuid.uuid1() 16 | self.exec_context = exec_context 17 | self.data = data 18 | self.next_pipes: set['Pipe'] = set() 19 | self.on_created() 20 | 21 | def plug_pipe(self, pipe: 'Pipe') -> 'Pipe': 22 | """ 23 | Plugs a pipe to the current pipe and returns the 24 | plugged pipe. 25 | """ 26 | self.next_pipes.add(pipe) 27 | return pipe 28 | 29 | def process_and_next(self, data: Any = None) -> Any: 30 | """ 31 | Process this pipe and process the plugged ones 32 | by giving them the result of this execution. 33 | """ 34 | result = self.process(data) 35 | for pipe in self.next_pipes: 36 | result = pipe.process_and_next(result) 37 | return result 38 | 39 | def __str__(self) -> str: 40 | return type(self).__name__ + ' ' + str(self.id) 41 | 42 | @abstractmethod 43 | def process(self, data: Any) -> Any: 44 | """ 45 | Contains the execution logic of this pipe. 46 | """ 47 | 48 | def on_created(self) -> None: 49 | """ 50 | This method is called when the pipe is created. 51 | 52 | It should be overriden by the child pipe if any action is needed 53 | at the creation. 54 | 55 | This is needed because child pipes can't have their own 56 | constructor since pipes are created dynamically. 57 | """ 58 | pass 59 | 60 | def extract_data(self, name: str, default: Any = None, required: bool = False) -> Any: 61 | """ 62 | Tries to get data from the data dictionary. 63 | 64 | If the data name provided exists in the dictionary it gets pop out and it gets returned. 65 | But if it doesn't exist, the default value provided is returned (None by default). 66 | 67 | if the required value is set to True and if the data can't be found, a YAMLSyntaxException is raised. 68 | """ 69 | if name in self.data: 70 | return self.data.pop(name) 71 | if not required: 72 | return default 73 | 74 | raise YAMLSyntaxException(f'The field "{name}" is required for the pipe of type {type(self).__name__}') 75 | 76 | def log(self, msg: str, level: int = logging.INFO) -> None: 77 | """ 78 | Log the given message with the given log level (default is INFO). 79 | The rule name is automatically prefixed to the log message. 80 | """ 81 | LOG.log(level, f'<{self.exec_context.rule_name if self.exec_context else "None"}> {msg}') 82 | -------------------------------------------------------------------------------- /src/nominatim_data_analyser/core/pipes/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module containing all pipes of the analyser. 3 | """ 4 | 5 | from .filling_pipe import FillingPipe as FillingPipe 6 | 7 | from .output_formatters.geojson_formatter import GeoJSONFormatter as GeoJSONFormatter 8 | from .output_formatters.geojson_feature_converter import GeoJSONFeatureConverter as GeoJSONFeatureConverter 9 | from .output_formatters.vector_tile_formatter import VectorTileFormatter as VectorTileFormatter 10 | from .output_formatters.osmoscope_layer_formatter import OsmoscopeLayerFormatter as OsmoscopeLayerFormatter 11 | from .output_formatters.clusters_vt_formatter import ClustersVtFormatter as ClustersVtFormatter 12 | 13 | from .data_fetching.sql_processor import SQLProcessor as SQLProcessor 14 | 15 | from .rules_specific_pipes.addr_house_number_no_digit.digits_filter import AddrHouseNumberNoDigitFilter as AddrHouseNumberNoDigitFilter 16 | from .rules_specific_pipes.duplicate_label_role.custom_feature_converter import DuplicateLabelRoleCustomFeatureConverter as DuplicateLabelRoleCustomFeatureConverter 17 | from .rules_specific_pipes.place_nodes_close.custom_feature_converter import PlaceNodesCloseCustomFeatureConverter as PlaceNodesCloseCustomFeatureConverter 18 | from .rules_specific_pipes.same_wikidata.custom_feature_converter import SameWikiDataFeatureConverter as SameWikiDataFeatureConverter 19 | 20 | from .data_processing.loop_data_processor import LoopDataProcessor as LoopDataProcessor 21 | from .data_processing.geometry_converter import GeometryConverter as GeometryConverter 22 | -------------------------------------------------------------------------------- /src/nominatim_data_analyser/core/pipes/data_fetching/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module handling the logic used to fetch and use 3 | the data in a QA rule. 4 | """ 5 | 6 | from .sql_processor import SQLProcessor -------------------------------------------------------------------------------- /src/nominatim_data_analyser/core/pipes/data_fetching/sql_processor.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | 3 | import psycopg 4 | 5 | from ....config import Config 6 | from ... import Pipe 7 | from ....logger.timer import Timer 8 | 9 | 10 | class SQLProcessor(Pipe): 11 | """ 12 | Handles the execution of an SQL query and 13 | send the results to the next pipe. 14 | """ 15 | def on_created(self) -> None: 16 | self.query = self.extract_data('query', required=True) 17 | 18 | def process(self, _: Any) -> list[dict[str, Any]]: 19 | """ 20 | Executes the query and returns the results. 21 | """ 22 | results: list[dict[str, Any]] 23 | 24 | with psycopg.connect(Config.values['Dsn']) as conn: 25 | with conn.cursor(row_factory=psycopg.rows.dict_row) as cur: 26 | timer = Timer().start_timer() 27 | cur.execute(self.query) 28 | results = cur.fetchall() 29 | elapsed_mins, elapsed_secs = timer.get_elapsed() 30 | self.log(f'Query {self.id} executed in {elapsed_mins} mins {elapsed_secs} secs.') 31 | self.log(f'Query {self.id} returned {len(results)} results.') 32 | 33 | return results 34 | -------------------------------------------------------------------------------- /src/nominatim_data_analyser/core/pipes/data_processing/__init__.py: -------------------------------------------------------------------------------- 1 | from .loop_data_processor import LoopDataProcessor 2 | from .geometry_converter import GeometryConverter -------------------------------------------------------------------------------- /src/nominatim_data_analyser/core/pipes/data_processing/geometry_converter.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | from ... import Pipe 3 | from ... import model as core_model 4 | 5 | class GeometryConverter(Pipe): 6 | """ 7 | Pipe used to convert data to a Geometry class. 8 | """ 9 | def on_created(self) -> None: 10 | self.geometry_type = self.extract_data('geometry_type', required=True) 11 | 12 | def process(self, data: dict[str, Any]) -> dict[str, Any] | None: 13 | """ 14 | Converts the given Well-Known Text representation of a 15 | geometry into a Geometry class based on the geometry_type. 16 | """ 17 | # If one data doesn't contain a geometry_holder it should be ignored. 18 | if (data['geometry_holder'] is None): 19 | return None 20 | dclass = getattr(core_model, self.geometry_type) 21 | convert = getattr(dclass, 'create_from_WKT_string') 22 | data['geometry_holder'] = convert(data['geometry_holder']) 23 | return data 24 | -------------------------------------------------------------------------------- /src/nominatim_data_analyser/core/pipes/data_processing/loop_data_processor.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | from typing import List, Any 3 | from ....logger.timer import Timer 4 | from ... import Pipe 5 | 6 | class LoopDataProcessor(Pipe): 7 | """ 8 | Transforms each element of a list by processing them with a 9 | custom processing pipeline. 10 | """ 11 | def on_created(self) -> None: 12 | self.processing_pipeline: Pipe = self.extract_data('sub_pipeline', required=True) 13 | 14 | def process(self, data: List[Any]) -> List[Any]: 15 | """ 16 | Processes each data of the input list with the processing pipeline. 17 | """ 18 | timer = Timer().start_timer() 19 | processed_data = list() 20 | for d in data: 21 | processed_result = self.process_one_data(d) 22 | if processed_result: 23 | # The result can be a list with multiple results or only one result 24 | if isinstance(processed_result, List): 25 | processed_data.extend(processed_result) 26 | else: 27 | processed_data.append(processed_result) 28 | 29 | elapsed_mins, elapsed_secs = timer.get_elapsed() 30 | self.log(f'Loop data processor executed in {elapsed_mins} mins {elapsed_secs} secs.') 31 | return processed_data 32 | 33 | def process_one_data(self, data: Any) -> Any: 34 | """ 35 | Processes one data through each pipe of the processing pipeline. 36 | If one pipe returns None the process is stopped and None is returned. 37 | """ 38 | current_pipe: Pipe | None = self.processing_pipeline 39 | while current_pipe: 40 | data = current_pipe.process(data) 41 | if data is None: 42 | break 43 | current_pipe = next(iter(current_pipe.next_pipes), None) 44 | return data 45 | -------------------------------------------------------------------------------- /src/nominatim_data_analyser/core/pipes/filling_pipe.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | from ...core import Pipe 3 | 4 | class FillingPipe(Pipe): 5 | """ 6 | Pipe used only for filling. 7 | It doesn't do anything with data. 8 | """ 9 | def process(self, data: Any = None) -> Any: 10 | """ 11 | Contains the execution logic of this pipe. 12 | """ 13 | return data 14 | -------------------------------------------------------------------------------- /src/nominatim_data_analyser/core/pipes/output_formatters/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module containing the formatters for the outputs 3 | """ 4 | from .geojson_formatter import * 5 | from .geojson_feature_converter import * 6 | from .vector_tile_formatter import * 7 | from .osmoscope_layer_formatter import * 8 | from .clusters_vt_formatter import * -------------------------------------------------------------------------------- /src/nominatim_data_analyser/core/pipes/output_formatters/clusters_vt_formatter.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | from geojson.feature import Feature, FeatureCollection 3 | from geojson import dumps 4 | from ....logger.timer import Timer 5 | from ....config import Config 6 | from ... import Pipe 7 | from ....clustering_vt import cluster # type: ignore[import-not-found] 8 | from pathlib import Path 9 | from typing import List 10 | 11 | class ClustersVtFormatter(Pipe): 12 | """ 13 | Handles the creation of the clusters and vector tiles. 14 | """ 15 | def on_created(self) -> None: 16 | self.base_folder_path = Path(f'{Config.values["RulesFolderPath"]}/{self.exec_context.rule_name}/vector-tiles') 17 | self.radius: int = self.extract_data('radius', default=60) 18 | 19 | def process(self, features: List[Feature]) -> str: 20 | """ 21 | Converts a list of GeoJSON features to clusters vector tiles by 22 | calling clustering-vt from the command line. 23 | 24 | The outputfolder is initially deleted if it exists. 25 | """ 26 | feature_collection = FeatureCollection(features) 27 | timer = Timer().start_timer() 28 | 29 | self.call_clustering_vt(self.base_folder_path, feature_collection) 30 | 31 | elapsed_mins, elapsed_secs = timer.get_elapsed() 32 | self.log(f'Clustering and vector tiles creation executed in {elapsed_mins} mins {elapsed_secs} secs') 33 | 34 | web_path = f'{Config.values["WebPrefixPath"]}/{self.exec_context.rule_name}/vector-tiles/' + '{z}/{x}/{y}.pbf' 35 | return web_path 36 | 37 | def call_clustering_vt(self, output_dir: Path, feature_collection: FeatureCollection) -> None: 38 | """ 39 | Calls clustering-vt through a subprocess and send the feature collection as a stream 40 | in the stdin of the subprocess. 41 | """ 42 | result = cluster(str(output_dir), self.radius, (dumps(feat) for feat in feature_collection['features'])) 43 | if result != 0: 44 | raise RuntimeError("Clustering failed.") 45 | -------------------------------------------------------------------------------- /src/nominatim_data_analyser/core/pipes/output_formatters/geojson_feature_converter.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | from ...dynamic_value import resolve_all 3 | from ... import Pipe 4 | from geojson import Feature 5 | 6 | class GeoJSONFeatureConverter(Pipe): 7 | """ 8 | Handle the conversion of generic data class 9 | to geojson features. 10 | """ 11 | def on_created(self) -> None: 12 | self.properties_pattern = self.extract_data('properties', default={}) 13 | self.current_id = -1 14 | 15 | def process(self, elements: dict[str, Any]) -> Feature: 16 | """ 17 | Convert a query result to a geojson feature. 18 | """ 19 | properties: dict[str, Any] = dict() 20 | self.current_id += 1 21 | if self.properties_pattern: 22 | for prop in self.properties_pattern: 23 | # Resolve dynamic values 24 | resolved_value = resolve_all(prop, elements) 25 | for k, v in resolved_value.items(): 26 | properties[k] = v 27 | returned_geom = elements.pop('geometry_holder').to_geojson_feature(self.current_id, properties) 28 | return returned_geom 29 | -------------------------------------------------------------------------------- /src/nominatim_data_analyser/core/pipes/output_formatters/geojson_formatter.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | from ....config import Config 3 | from typing import List 4 | from geojson.feature import Feature 5 | from ....core import Pipe 6 | from pathlib import Path 7 | from geojson import FeatureCollection, dump 8 | 9 | class GeoJSONFormatter(Pipe): 10 | """ 11 | Handles the creation of the GeoJSON file. 12 | """ 13 | def on_created(self) -> None: 14 | self.base_folder_path = Path(f'{Config.values["RulesFolderPath"]}/{self.exec_context.rule_name}/geojson') 15 | # Take the rule's name as default file name. 16 | self.file_name = self.extract_data('file_name', self.exec_context.rule_name) 17 | 18 | def process(self, features: List[Feature]) -> str: 19 | """ 20 | Create the FeatureCollection and dump it to 21 | a new GeoJSON file. 22 | """ 23 | feature_collection = FeatureCollection(features) 24 | self.base_folder_path.mkdir(parents=True, exist_ok=True) 25 | full_path = self.base_folder_path / f'{self.file_name}.json' 26 | 27 | with open(full_path, 'w') as file: 28 | dump(feature_collection, file) 29 | 30 | web_path = f'{Config.values["WebPrefixPath"]}/{self.exec_context.rule_name}/geojson/{self.file_name}.json' 31 | return web_path 32 | -------------------------------------------------------------------------------- /src/nominatim_data_analyser/core/pipes/output_formatters/osmoscope_layer_formatter.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | import json 3 | 4 | import psycopg 5 | 6 | from ....config import Config 7 | from ... import Pipe 8 | 9 | class OsmoscopeLayerFormatter(Pipe): 10 | """ 11 | Handles the creation of the layer JSON file. 12 | """ 13 | def on_created(self) -> None: 14 | self.base_folder_path = Path(f'{Config.values["RulesFolderPath"]}/{self.exec_context.rule_name}/osmoscope-layer') 15 | self.file_name = self.extract_data('file_name', 'layer') 16 | self.data_format_url = self.extract_data('data_format_url', required=True) 17 | self.data['id'] = self.extract_data('id', default=self.exec_context.rule_name) 18 | 19 | def process(self, data_source_path: str) -> None: 20 | """ 21 | Create the JSON layer file containing the right data. 22 | 23 | It gets the GeoJSON url as data parameter and set it 24 | inside the layer file. 25 | """ 26 | self.add_last_update_date_layer_info() 27 | self.data[self.data_format_url] = data_source_path 28 | self.base_folder_path.mkdir(parents=True, exist_ok=True) 29 | full_path = self.base_folder_path / f'{self.file_name}.json' 30 | 31 | with open(full_path, 'w') as json_file: 32 | json.dump(self.data, json_file) 33 | 34 | file_url = f'{Config.values["WebPrefixPath"]}/{self.exec_context.rule_name}/osmoscope-layer/{self.file_name}.json' 35 | self.add_layer_to_global_layers_file(file_url) 36 | 37 | def add_last_update_date_layer_info(self) -> None: 38 | """ 39 | Add a "last_update" field to the layer information. 40 | This field contains the date of the last database update. 41 | The date is extracted from the lastimportdate table of the database. 42 | """ 43 | with psycopg.connect(Config.values['Dsn']) as conn: 44 | with conn.cursor() as cur: 45 | cur.execute("SELECT to_char(lastimportdate at time zone 'UTC', " 46 | " 'YYYY-MM-DD HH24:MI:SS UTC') FROM import_status") 47 | last_update_date = cur.fetchone() 48 | if last_update_date: 49 | if 'doc' not in self.data: 50 | self.data['doc'] = {} 51 | self.data['doc']['last_update'] = last_update_date[0] 52 | 53 | def add_layer_to_global_layers_file(self, path: str) -> None: 54 | """ 55 | Add the newly created layer to the global layers file. 56 | If the global layers file doesn't exist it is created. 57 | """ 58 | folder_path = Path(f'{Config.values["RulesFolderPath"]}') 59 | folder_path.mkdir(parents=True, exist_ok=True) 60 | # Check if the folder_path has a parent because /layers.json will require sudo permissions. 61 | full_path = folder_path / 'layers.json' if len(folder_path.parents) > 0 else Path('layers.json') 62 | full_path.touch(exist_ok=True) 63 | 64 | with open(full_path, 'r') as json_file: 65 | try: 66 | data = json.load(json_file) 67 | except: 68 | data = { 69 | 'name': 'Nominatim suspects', 70 | 'layers': [] 71 | } 72 | if path not in data['layers']: 73 | data['layers'].append(path) 74 | with open(full_path, 'w') as json_file: 75 | json.dump(data, json_file) 76 | -------------------------------------------------------------------------------- /src/nominatim_data_analyser/core/pipes/output_formatters/vector_tile_formatter.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | from geojson.feature import Feature, FeatureCollection 3 | from geojson import dumps 4 | from ....logger.timer import Timer 5 | from ....config import Config 6 | from ... import Pipe 7 | from pathlib import Path 8 | from typing import List 9 | import subprocess 10 | import logging 11 | 12 | class VectorTileFormatter(Pipe): 13 | """ 14 | Handles the creation of the Vector tiles. 15 | """ 16 | def on_created(self) -> None: 17 | self.base_folder_path = Path(f'{Config.values["RulesFolderPath"]}/{self.exec_context.rule_name}/vector-tiles') 18 | 19 | def process(self, features: List[Feature]) -> str: 20 | """ 21 | Converts a list of features to vector tiles by 22 | calling tippecanoe from the command line. 23 | """ 24 | feature_collection = FeatureCollection(features) 25 | self.base_folder_path.mkdir(parents=True, exist_ok=True) 26 | timer = Timer().start_timer() 27 | 28 | self.call_tippecanoe(self.base_folder_path, feature_collection) 29 | 30 | elapsed_mins, elapsed_secs = timer.get_elapsed() 31 | self.log(f'Vector tile conversion executed in {elapsed_mins} mins {elapsed_secs} secs') 32 | 33 | web_path = f'{Config.values["WebPrefixPath"]}/{self.exec_context.rule_name}/vector-tiles/' + '{z}/{x}/{y}.pbf' 34 | return web_path 35 | 36 | def call_tippecanoe(self, output_dir: Path, feature_collection: FeatureCollection) -> None: 37 | """ 38 | Calls Tippecanoe through a subprocess and send the feature collection as a stream 39 | in the stdin of the subprocess. 40 | """ 41 | try: 42 | result = subprocess.run( 43 | ['tippecanoe', f'--output-to-directory={output_dir}', 44 | '--force', 45 | '--no-tile-compression', 46 | '--no-tile-size-limit', 47 | '--no-feature-limit', 48 | '--buffer=125', 49 | '--no-clipping', 50 | '-r1', 51 | '--cluster-distance=60'], 52 | check=True, 53 | input=dumps(feature_collection).encode(), 54 | stdout=subprocess.PIPE 55 | ) 56 | self.log(str(result)) 57 | except subprocess.TimeoutExpired as e: 58 | self.log(str(e), logging.FATAL) 59 | except subprocess.CalledProcessError as e: 60 | self.log(str(e), logging.FATAL) 61 | -------------------------------------------------------------------------------- /src/nominatim_data_analyser/core/pipes/rules_specific_pipes/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module contains custom advanced logical units. 3 | """ 4 | 5 | from .addr_house_number_no_digit.digits_filter import * 6 | from .duplicate_label_role.custom_feature_converter import * 7 | from .place_nodes_close.custom_feature_converter import * 8 | from .same_wikidata.custom_feature_converter import * 9 | -------------------------------------------------------------------------------- /src/nominatim_data_analyser/core/pipes/rules_specific_pipes/addr_house_number_no_digit/digits_filter.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | from ....pipe import Pipe 3 | import re 4 | 5 | class AddrHouseNumberNoDigitFilter(Pipe): 6 | def on_created(self) -> None: 7 | self.any_digit_regex = re.compile(r'.*\d.*') 8 | 9 | def process(self, elements: dict[str, Any]) -> dict[str, Any] | None: 10 | """ 11 | Filter the given data result by checking if 12 | the housenumber contains any digit in any scripts 13 | (by using the \\d in python regex). 14 | """ 15 | # Keep only data where we do not match at least one digit 16 | if not self.any_digit_regex.match(elements['housenumber']): 17 | return elements 18 | return None 19 | -------------------------------------------------------------------------------- /src/nominatim_data_analyser/core/pipes/rules_specific_pipes/duplicate_label_role/custom_feature_converter.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | from ....pipe import Pipe 3 | from geojson.feature import Feature 4 | 5 | class DuplicateLabelRoleCustomFeatureConverter(Pipe): 6 | def on_created(self) -> None: 7 | self.current_feature_id = -1 8 | 9 | def process(self, elements: dict[str, Any]) -> Feature: 10 | """ 11 | Creates Geojson features for the given result of the SQLProcessor. 12 | 13 | Extract members with role=label from the list of members 14 | returned by the table planet_osm_rels to display them in 15 | the properties of the Node. 16 | 17 | The members array has the following syntax: 18 | ['w8125151','outer','w249285853','inner'.......] 19 | It works by pair where the first item ('w8125151' for example) 20 | contains the object type (n, w, r) and its osm_id, then 21 | the second item is the role ('outer' for example). 22 | """ 23 | self.current_feature_id += 1 24 | properties = { 25 | 'relation_id': elements['osm_id'] 26 | } 27 | members = elements['members'] 28 | label_members_count = 0 29 | for i in range(len(members) - 1): 30 | role = members[i+1] 31 | if role == 'label': 32 | label_members_count += 1 33 | # Get the n/w/r type 34 | type = members[i][0] 35 | properties[f'{type}/@idLabel {label_members_count}'] = members[i][1:] 36 | 37 | return elements.pop('geometry_holder').to_geojson_feature(self.current_feature_id, properties) 38 | -------------------------------------------------------------------------------- /src/nominatim_data_analyser/core/pipes/rules_specific_pipes/place_nodes_close/custom_feature_converter.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | from .....core.pipe import Pipe 3 | from geojson.feature import Feature 4 | 5 | class PlaceNodesCloseCustomFeatureConverter(Pipe): 6 | def on_created(self) -> None: 7 | self.current_feature_id = -1 8 | 9 | def process(self, elements: dict[str, Any]) -> Feature: 10 | """ 11 | Creates a Geojson feature for the given elements dictionnary. 12 | Adds a specific property for each id in the 'common_ids' field. 13 | """ 14 | self.current_feature_id += 1 15 | 16 | properties = { 17 | 'node_id': elements['osm_id'] 18 | } 19 | common_ids = elements['common_ids'] 20 | for i in range(len(common_ids)): 21 | properties[f'n/@idClose node {i+1}'] = common_ids[i] 22 | 23 | return elements.pop('geometry_holder').to_geojson_feature(self.current_feature_id, properties) 24 | -------------------------------------------------------------------------------- /src/nominatim_data_analyser/core/pipes/rules_specific_pipes/same_wikidata/custom_feature_converter.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | from ....pipe import Pipe 3 | from ....model.node import Node 4 | from geojson import Feature 5 | 6 | class SameWikiDataFeatureConverter(Pipe): 7 | def process(self, data: list[dict[str, Any]]) -> list[Feature]: 8 | """ 9 | Creates Geojson features for each nodes. 10 | each result from the SQLProcessor contains 11 | multiple nodes. 12 | 13 | data is a list of list of format: 14 | [wikidata, List[ids], List[centroids]] 15 | """ 16 | features: list[Feature] = list() 17 | current_feature_id = 0 18 | for record in data: 19 | for i, centroid in enumerate(record['centroids']): 20 | # If one centroid is None the data should be ignored. 21 | if (centroid is None): 22 | continue 23 | node = Node.create_from_WKT_string(centroid) 24 | nodes_in_common = list() 25 | # Fetch concerning node id and id of each other nodes with the 26 | # same wikidata 27 | for j, id in enumerate(record['ids']): 28 | if j == i: 29 | node_id = id 30 | else: 31 | nodes_in_common.append(id) 32 | properties = { 33 | 'node_id': node_id, 34 | 'wikidata in common': record['wikidata'] 35 | } 36 | for i, id in enumerate(nodes_in_common): 37 | properties['n/@idNode in common ' + str(i + 1)] = id 38 | features.append(node.to_geojson_feature(current_feature_id, properties)) 39 | current_feature_id += 1 40 | return features 41 | -------------------------------------------------------------------------------- /src/nominatim_data_analyser/core/qa_rule/__init__.py: -------------------------------------------------------------------------------- 1 | from .execution_context import ExecutionContext as ExecutionContext 2 | -------------------------------------------------------------------------------- /src/nominatim_data_analyser/core/qa_rule/execution_context.py: -------------------------------------------------------------------------------- 1 | class ExecutionContext(): 2 | """ 3 | Execution context of a QA Rule pipeline 4 | which contains data and objects. 5 | 6 | It is stored in each pipe of the QA Rule pipeline. 7 | """ 8 | def __init__(self) -> None: 9 | self.rule_name = '' 10 | -------------------------------------------------------------------------------- /src/nominatim_data_analyser/core/yaml_logic/__init__.py: -------------------------------------------------------------------------------- 1 | from .yaml_loader import load_yaml_rule -------------------------------------------------------------------------------- /src/nominatim_data_analyser/core/yaml_logic/yaml_loader.py: -------------------------------------------------------------------------------- 1 | from typing import Any, cast 2 | from pathlib import Path 3 | 4 | import yaml 5 | 6 | from ..dynamic_value.switch import Switch 7 | from ..dynamic_value.variable import Variable 8 | from ..assembler import PipelineAssembler 9 | from .. import Pipe 10 | from ...logger.logger import LOG 11 | 12 | def load_yaml_rule(rule_file: Path) -> dict[str, Any]: 13 | """ 14 | Load the YAML specification file. 15 | YAML constructors are added to handle custom types in the YAML. 16 | """ 17 | def _sub_pipeline(loader: yaml.SafeLoader, node: yaml.Node) -> Pipe: 18 | if not isinstance(node, yaml.MappingNode): 19 | raise RuntimeError("!switch expects mapping.") 20 | return sub_pipeline_constructor(loader, node, rule_file.stem) 21 | 22 | yaml.add_constructor(u'!sub-pipeline', _sub_pipeline, Loader=yaml.SafeLoader) 23 | yaml.add_constructor(u'!variable', variable_constructor, Loader=yaml.SafeLoader) 24 | yaml.add_constructor(u'!switch', switch_constructor, Loader=yaml.SafeLoader) 25 | 26 | with rule_file.open('r') as file: 27 | try: 28 | loaded = cast(dict[str, Any], yaml.safe_load(file)) 29 | except yaml.YAMLError as exc: 30 | LOG.error('Error while loading the YAML rule file %s: %s', 31 | rule_file.stem, exc) 32 | raise 33 | 34 | return loaded 35 | 36 | def sub_pipeline_constructor(loader: yaml.SafeLoader, node: yaml.MappingNode, 37 | rule_name: str) -> Pipe: 38 | """ 39 | Loads the pipeline specification from the YAML node and 40 | assembles a pipeline with the PipelineAssembler. 41 | 42 | This constructor is used for the !sub-pipeline custom type. 43 | """ 44 | pipeline_specification = cast(dict[str, Any], loader.construct_mapping(node, deep=True)) 45 | return PipelineAssembler(pipeline_specification, rule_name).assemble() 46 | 47 | def variable_constructor(loader: yaml.SafeLoader, node: yaml.Node) -> Variable: 48 | """ 49 | Creates a Variable object using the node's data. 50 | """ 51 | if not isinstance(node, yaml.ScalarNode): 52 | raise RuntimeError("!variable expects scalar value.") 53 | 54 | return Variable(loader.construct_scalar(node)) 55 | 56 | def switch_constructor(loader: yaml.SafeLoader, node: yaml.Node) -> Switch: 57 | """ 58 | Creates a Switch object using the node's data. 59 | """ 60 | if not isinstance(node, yaml.MappingNode): 61 | raise RuntimeError("!switch expects mapping.") 62 | 63 | data = loader.construct_mapping(node, deep=True) 64 | return Switch(data['expression'], data['cases']) 65 | -------------------------------------------------------------------------------- /src/nominatim_data_analyser/default_config.yaml: -------------------------------------------------------------------------------- 1 | # Data source name for the database connection. 2 | Dsn: 'dbname=nominatim' 3 | 4 | # Path to the folder where rules data are stored (geojson, vector tiles, etc). 5 | RulesFolderPath: 'qa-data' 6 | 7 | # Prefix path of the web URL to access the rules data (ex: https://nominatim.org/QA-data). 8 | WebPrefixPath: '' 9 | 10 | -------------------------------------------------------------------------------- /src/nominatim_data_analyser/logger/logger.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | logging.basicConfig(format='%(asctime)s: %(message)s', 4 | datefmt='%Y-%m-%d %H:%M:%S') 5 | LOG = logging.getLogger() 6 | LOG.setLevel(logging.INFO) 7 | -------------------------------------------------------------------------------- /src/nominatim_data_analyser/logger/timer.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | from typing import Tuple 3 | from time import time 4 | 5 | class Timer(): 6 | def __init__(self) -> None: 7 | self.start_time = 0.0 8 | 9 | def start_timer(self) -> Timer: 10 | self.start_time = time() 11 | return self 12 | 13 | def get_elapsed(self) -> Tuple[int, int]: 14 | hours, rem = divmod(time() - self.start_time, 3600) 15 | minutes, seconds = divmod(rem, 60) 16 | return int(round(minutes, 1)), int(round(seconds, 1)) 17 | -------------------------------------------------------------------------------- /src/nominatim_data_analyser/py.typed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/osm-search/Nominatim-Data-Analyser/57cd6060ab506e3f7bf0b0e50f1fd1aa25e91567/src/nominatim_data_analyser/py.typed -------------------------------------------------------------------------------- /src/nominatim_data_analyser/rules_specifications/BA_way_not_part_relation.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | QUERY: 3 | type: SQLProcessor 4 | query: > 5 | SELECT ST_AsText(centroid) AS geometry_holder, osm_id FROM ( 6 | SELECT centroid, osm_id, array_agg(osm_id) AS array_osm_id FROM placex 7 | WHERE osm_type='W' AND class='boundary' AND type='administrative' GROUP BY osm_id, centroid 8 | ) AS px 9 | WHERE NOT EXISTS(SELECT 1 FROM planet_osm_rels WHERE px.array_osm_id <@ planet_osm_member_ids(members, 'W'::char(1))); 10 | out: 11 | LOOP_PROCESSING: 12 | type: LoopDataProcessor 13 | sub_pipeline: !sub-pipeline 14 | GEOMETRY_CONVERTER: 15 | type: GeometryConverter 16 | geometry_type: Node 17 | out: 18 | FEATURE_CONVERTER: 19 | type: GeoJSONFeatureConverter 20 | properties: 21 | - way_id: !variable osm_id 22 | out: 23 | CLUSTERING_VECTOR_TILES: 24 | type: ClustersVtFormatter 25 | radius: 60 26 | out: 27 | LAYER_FILE: 28 | type: OsmoscopeLayerFormatter 29 | data_format_url: vector_tile_url 30 | name: boundary ways without relation 31 | updates: Every evening 32 | doc: 33 | description: Ways with boundary=administrative that are not part of a relation. 34 | why_problem: Administrative boundary should always be mapped as relations. Every boundary way should be member of a relation. 35 | how_to_fix: You might have to create an appropriate administrative relation. 36 | -------------------------------------------------------------------------------- /src/nominatim_data_analyser/rules_specifications/addr_housenumber_no_digit.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | QUERY: 3 | type: SQLProcessor 4 | query: > 5 | SELECT ST_AsText(centroid) AS geometry_holder, osm_id, osm_type, address, address->'housenumber' as housenumber 6 | FROM placex 7 | WHERE address ? 'housenumber' AND not address ? '_inherited' 8 | AND address->'housenumber' NOT SIMILAR TO '%[[:digit:]]%' 9 | AND lower(address->'housenumber') NOT IN ('s/n', 'sn', 'bb'); 10 | out: 11 | LOOP_PROCESSING: 12 | type: LoopDataProcessor 13 | sub_pipeline: !sub-pipeline 14 | NO_DIGIT_FILTER: 15 | type: AddrHouseNumberNoDigitFilter 16 | out: 17 | GEOMETRY_CONVERTER: 18 | type: GeometryConverter 19 | geometry_type: Node 20 | out: 21 | FEATURE_CONVERTER: 22 | type: GeoJSONFeatureConverter 23 | properties: 24 | - !switch 25 | expression: osm_type 26 | cases: 27 | 'N': 28 | node_id: !variable osm_id 29 | 'W': 30 | way_id: !variable osm_id 31 | 'R': 32 | relation_id: !variable osm_id 33 | - address: !variable address 34 | out: 35 | CLUSTERING_VECTOR_TILES: 36 | type: ClustersVtFormatter 37 | radius: 60 38 | out: 39 | LAYER_FILE: 40 | type: OsmoscopeLayerFormatter 41 | data_format_url: vector_tile_url 42 | name: addr:housenumber with no digit 43 | updates: Every evening 44 | doc: 45 | description: addr:housenumber without any digit in them. 46 | why_problem: Normally house number are numbers. So, this layer tries to detect mapping mistakes by looking for house numbers that have no digits in them. 47 | how_to_fix: Check if the house number is correct or make sense. Either fix it or delete it. 48 | -------------------------------------------------------------------------------- /src/nominatim_data_analyser/rules_specifications/addr_place_and_street.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | QUERY: 3 | type: SQLProcessor 4 | query: > 5 | SELECT ST_AsText(centroid) AS geometry_holder, osm_id, osm_type, address FROM placex 6 | WHERE not address ? '_inherited' 7 | AND address ? 'place' AND address ? 'street' AND NOT address ? 'conscriptionnumber'; 8 | out: 9 | LOOP_PROCESSING: 10 | type: LoopDataProcessor 11 | sub_pipeline: !sub-pipeline 12 | GEOMETRY_CONVERTER: 13 | type: GeometryConverter 14 | geometry_type: Node 15 | out: 16 | FEATURE_CONVERTER: 17 | type: GeoJSONFeatureConverter 18 | properties: 19 | - !switch 20 | expression: osm_type 21 | cases: 22 | 'N': 23 | node_id: !variable osm_id 24 | 'W': 25 | way_id: !variable osm_id 26 | 'R': 27 | relation_id: !variable osm_id 28 | - address: !variable address 29 | out: 30 | CLUSTERING_VECTOR_TILES: 31 | type: ClustersVtFormatter 32 | radius: 60 33 | out: 34 | LAYER_FILE: 35 | type: OsmoscopeLayerFormatter 36 | data_format_url: vector_tile_url 37 | name: addr:place vs addr:street 38 | updates: Every evening 39 | doc: 40 | description: addr:place and addr:street on the same object. 41 | why_problem: Usually an house number refers either to a street or to a larger place. addr:place and addr:street should therefore not be used together. 42 | how_to_fix: Set only addr:place or addr:street but not both on the same object. Usually addr:place should be changed to something like addr:suburb or addr:city. 43 | -------------------------------------------------------------------------------- /src/nominatim_data_analyser/rules_specifications/addr_place_or_street_rank28.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | QUERY: 3 | type: SQLProcessor 4 | query: > 5 | SELECT ST_AsText(centroid) AS geometry_holder, osm_id, osm_type, address, class, type 6 | FROM placex p WHERE (address ? 'street' or address ? 'place') 7 | AND rank_search < 28 8 | AND NOT (class in ('landuse') 9 | OR type in ('postcode', 'houses') 10 | OR (class = 'leisure' and type = 'park')) 11 | AND NOT EXISTS (SELECT * FROM placex o 12 | WHERE o.osm_id = p.osm_id and o.osm_type = p.osm_type and rank_address = 30); 13 | out: 14 | LOOP_PROCESSING: 15 | type: LoopDataProcessor 16 | sub_pipeline: !sub-pipeline 17 | GEOMETRY_CONVERTER: 18 | type: GeometryConverter 19 | geometry_type: Node 20 | out: 21 | FEATURE_CONVERTER: 22 | type: GeoJSONFeatureConverter 23 | properties: 24 | - key: !variable class 25 | - value: !variable type 26 | - !switch 27 | expression: osm_type 28 | cases: 29 | 'N': 30 | node_id: !variable osm_id 31 | 'W': 32 | way_id: !variable osm_id 33 | 'R': 34 | relation_id: !variable osm_id 35 | - address: !variable address 36 | out: 37 | CLUSTERING_VECTOR_TILES: 38 | type: ClustersVtFormatter 39 | radius: 60 40 | out: 41 | LAYER_FILE: 42 | type: OsmoscopeLayerFormatter 43 | data_format_url: vector_tile_url 44 | name: addr:* tags on non-addressable places 45 | updates: Every evening 46 | doc: 47 | description: addr:street or addr:place on objects that normally do not have an address. 48 | how_to_fix: | 49 | Check if the object really should have an address. Remove all tags 'addr:*'. 50 | If this is not the case or convert them to a more appropriate tag, 51 | for example using 'contact:*' tags. 52 | -------------------------------------------------------------------------------- /src/nominatim_data_analyser/rules_specifications/addr_street_wrong_name.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | QUERY: 3 | type: SQLProcessor 4 | query: > 5 | SELECT ST_AsText(px1.centroid) AS geometry_holder, px1.osm_id, px1.osm_type, px1.address->'street' as street_name, 6 | px2.name->'name' as parent_name FROM ( 7 | SELECT * FROM placex 8 | WHERE rank_address=30 AND address ? 'street' AND not address ? '_inherited' 9 | ) AS px1 INNER JOIN placex AS px2 10 | ON px1.parent_place_id = px2.place_id 11 | WHERE px1.address->'street' not in (SELECT regexp_split_to_table(svals(px2.name), '\s*;\s*')) 12 | and not lower(px1.address->'street') = lower(px2.name->'name'); 13 | out: 14 | LOOP_PROCESSING: 15 | type: LoopDataProcessor 16 | sub_pipeline: !sub-pipeline 17 | GEOMETRY_CONVERTER: 18 | type: GeometryConverter 19 | geometry_type: Node 20 | out: 21 | FEATURE_CONVERTER: 22 | type: GeoJSONFeatureConverter 23 | properties: 24 | - !switch 25 | expression: osm_type 26 | cases: 27 | 'N': 28 | node_id: !variable osm_id 29 | 'W': 30 | way_id: !variable osm_id 31 | 'R': 32 | relation_id: !variable osm_id 33 | - street_name: !variable street_name 34 | - parent_name: !variable parent_name 35 | out: 36 | CLUSTERING_VECTOR_TILES: 37 | type: ClustersVtFormatter 38 | radius: 60 39 | out: 40 | LAYER_FILE: 41 | type: OsmoscopeLayerFormatter 42 | data_format_url: vector_tile_url 43 | name: Suspicious addr:street tag 44 | updates: Every evening 45 | doc: 46 | description: | 47 | This view shows addresses where the addr:street tag differs 48 | from the name of the street that Nominatim has assigned to 49 | the address. There are three different reasons that this 50 | happens: 1) there is a typo in addr:street or the street name. 51 | 2) The addr:street part does not refer to a street at all 52 | but to a place (village, hamlet, area). 53 | 3) Road is not mapped yet near address (or tts name is missing) 54 | why_problem: | 55 | 1) The addr:street tag should have exactly the same name as the street. 56 | 2) addr:street must only be used when the house number is attached 57 | to an existing street nearby. 58 | how_to_fix: | 59 | 1) Check if there is a typo in the addr:street tag or 60 | in the name of the street and fix it. 61 | 2) If addr:street refers to a place, use change it to addr:place. 62 | 3) If road is missing, map it 63 | -------------------------------------------------------------------------------- /src/nominatim_data_analyser/rules_specifications/bad_interpolations.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | QUERY: 3 | type: SQLProcessor 4 | query: > 5 | SELECT DISTINCT ON (osm_id) * FROM 6 | (SELECT 'failed to parse house numbers' as problem, osm_id, 7 | address->'interpolation' as interpolation, 8 | ST_AsText(ST_Centroid(linegeo)) as geometry_holder 9 | FROM location_property_osmline WHERE startnumber is null and indexed_status = 0 10 | UNION ALL 11 | SELECT 'bad addr:interpolation tag' as problem, osm_id, 12 | address->'interpolation' as interpolation, 13 | ST_AsText(ST_Centroid(geometry)) as geometry_holder 14 | FROM place 15 | WHERE address ? 'interpolation' 16 | and not (address->'interpolation' in ('even', 'odd', 'all', 'alphabetic') 17 | or address->'interpolation' similar to '[1-9]')) u 18 | out: 19 | LOOP_PROCESSING: 20 | type: LoopDataProcessor 21 | sub_pipeline: !sub-pipeline 22 | GEOMETRY_CONVERTER: 23 | type: GeometryConverter 24 | geometry_type: Node 25 | out: 26 | FEATURE_CONVERTER: 27 | type: GeoJSONFeatureConverter 28 | properties: 29 | - problem: !variable problem 30 | - way_id: !variable osm_id 31 | - interpolation_tag: !variable interpolation 32 | out: 33 | CLUSTERING_VECTOR_TILES: 34 | type: ClustersVtFormatter 35 | radius: 60 36 | out: 37 | LAYER_FILE: 38 | type: OsmoscopeLayerFormatter 39 | data_format_url: vector_tile_url 40 | name: Bad interpolation line 41 | updates: Every evening 42 | doc: 43 | description: | 44 | This view shows address interpolation lines that are 45 | problematic. Either the addr:interpolation tag doesn't have 46 | one of the known values: all, even, odd, alphabetic or a 47 | number. Or the address nodes on the line that describe the 48 | start and end values to use for the interpolation cannot be 49 | parsed. 50 | why_problem: | 51 | To process an interpolation, Nominatim needs numerical 52 | housenumber as the start and end value and it needs to know 53 | how to fill the space between the numbers (with odd, even 54 | or any number). The view also shows unnecessary interpolations. 55 | These are interpolation without a missing number. For example 56 | an interpolation starting at 1 and ending at 3 for odd numbers 57 | would never produce additional number between 1 and 3. 58 | how_to_fix: | 59 | Check that the addr:interpolation tag and change it to one 60 | of the legal values, if it is wrong. Check that the 61 | interpolation line has a node with addr:housenumber at the 62 | beginning and end and check that the values are senisble. 63 | -------------------------------------------------------------------------------- /src/nominatim_data_analyser/rules_specifications/duplicate_label_role.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | QUERY: 3 | type: SQLProcessor 4 | query: > 5 | SELECT ST_AsText(centroid) AS geometry_holder, osm_id, osm_type, members FROM placex AS px JOIN planet_osm_rels AS por ON px.osm_id=por.id 6 | WHERE class='boundary' AND type='administrative' AND 7 | jsonb_array_length(jsonb_path_query_array(members, '$[*].role ? (@ == "label")')) >= 2; 8 | out: 9 | LOOP_PROCESSING: 10 | type: LoopDataProcessor 11 | sub_pipeline: !sub-pipeline 12 | GEOMETRY_CONVERTER: 13 | type: GeometryConverter 14 | geometry_type: Node 15 | out: 16 | FEATURE_CONVERTER: 17 | type: DuplicateLabelRoleCustomFeatureConverter 18 | out: 19 | GEOJSON: 20 | type: GeoJSONFormatter 21 | out: 22 | LAYER_FILE: 23 | type: OsmoscopeLayerFormatter 24 | data_format_url: geojson_url 25 | name: duplicate label role 26 | updates: Every evening 27 | doc: 28 | description: Admin boundaries with more than one member with role 'label'. 29 | why_problem: Boundary can have exactly one member with place label. 30 | how_to_fix: Make sure that the admin boundary contains only one member with role 'label'. 31 | -------------------------------------------------------------------------------- /src/nominatim_data_analyser/rules_specifications/no_admin_level.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | QUERY: 3 | type: SQLProcessor 4 | query: > 5 | SELECT ST_AsText(centroid) AS geometry_holder, osm_id FROM placex WHERE osm_type='R' AND class='boundary' 6 | AND type='administrative' AND admin_level >= 15; 7 | out: 8 | LOOP_PROCESSING: 9 | type: LoopDataProcessor 10 | sub_pipeline: !sub-pipeline 11 | GEOMETRY_CONVERTER: 12 | type: GeometryConverter 13 | geometry_type: Node 14 | out: 15 | FEATURE_CONVERTER: 16 | type: GeoJSONFeatureConverter 17 | properties: 18 | - relation_id: !variable osm_id 19 | out: 20 | GEOJSON: 21 | type: GeoJSONFormatter 22 | out: 23 | LAYER_FILE: 24 | type: OsmoscopeLayerFormatter 25 | data_format_url: geojson_url 26 | name: no admin level 27 | updates: Every evening 28 | doc: 29 | description: Every relation with boundary=administrative should have an admin_level value set. 30 | why_problem: Check the [wiki page](https://wiki.openstreetmap.org/wiki/Tag%3Aboundary%3Dadministrative) for more informations. 31 | how_to_fix: Add a tag 'admin_level' to the boundary relation. -------------------------------------------------------------------------------- /src/nominatim_data_analyser/rules_specifications/place_nodes_close.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | QUERY: 3 | type: SQLProcessor 4 | query: > 5 | SELECT ST_AsText(px1.centroid) AS geometry_holder, px1.osm_id as osm_id, ARRAY_AGG(px2.osm_id) AS common_ids 6 | FROM (SELECT * FROM placex WHERE osm_type='N' AND class='place' AND type != 'postcode' AND rank_search < 26) AS px1 7 | JOIN (SELECT * FROM placex WHERE osm_type='N' AND class='place' AND type != 'postcode' AND rank_search < 26) AS px2 8 | ON px1.type = px2.type and px1.name = px2.name and px1.osm_id != px2.osm_id 9 | WHERE ST_DistanceSphere(px1.centroid, px2.centroid) / 1000 <= 1 GROUP BY px1.osm_id, px1.centroid; 10 | out: 11 | LOOP_PROCESSING: 12 | type: LoopDataProcessor 13 | sub_pipeline: !sub-pipeline 14 | GEOMETRY_CONVERTER: 15 | type: GeometryConverter 16 | geometry_type: Node 17 | out: 18 | FEATURE_CONVERTER: 19 | type: PlaceNodesCloseCustomFeatureConverter 20 | out: 21 | CLUSTERING_VECTOR_TILES: 22 | type: ClustersVtFormatter 23 | radius: 60 24 | out: 25 | LAYER_FILE: 26 | type: OsmoscopeLayerFormatter 27 | data_format_url: vector_tile_url 28 | name: place nodes close 29 | updates: Every evening 30 | doc: 31 | description: Place nodes of same type and name close to each other. 32 | why_problem: Place nodes with the same type and name which are geographically close to each other are probably errors. 33 | how_to_fix: Remove or correct the duplicate nodes. -------------------------------------------------------------------------------- /src/nominatim_data_analyser/rules_specifications/same_wikidata.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | QUERY: 3 | type: SQLProcessor 4 | query: > 5 | SELECT wikidata, ids, centroids FROM (SELECT extratags->'wikidata' as wikidata, array_agg(osm_id) as ids, 6 | array_agg(ST_AsText(centroid)) as centroids, count(1) FROM placex 7 | WHERE class='place' and rank_search < 26 8 | AND osm_type='N' AND extratags ? 'wikidata' GROUP BY wikidata) foo WHERE count > 1 9 | out: 10 | CONVERTER: 11 | type: SameWikiDataFeatureConverter 12 | out: 13 | CLUSTERING_VECTOR_TILES: 14 | type: ClustersVtFormatter 15 | radius: 60 16 | out: 17 | LAYER_FILE: 18 | type: OsmoscopeLayerFormatter 19 | data_format_url: vector_tile_url 20 | name: duplicate wikidata id 21 | updates: Every evening 22 | doc: 23 | description: A single wikidata id should be assigned to at most one place node. 24 | why_problem: Wikidata object describes exactly one place, if the same wikidata id is found on multiple places then usually the id belongs to the parent place. 25 | how_to_fix: Check the wikidata entries, leave the wikidata id only on the place that corresponds exactly to the entry. Sometimes you have to remove all of the wikidata ids because they belong to the parent. 26 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/osm-search/Nominatim-Data-Analyser/57cd6060ab506e3f7bf0b0e50f1fd1aa25e91567/tests/__init__.py -------------------------------------------------------------------------------- /tests/assembler/test_pipe_factory.py: -------------------------------------------------------------------------------- 1 | from nominatim_data_analyser.core.pipes.data_processing.geometry_converter import GeometryConverter 2 | from nominatim_data_analyser.core.assembler.pipe_factory import PipeFactory 3 | from nominatim_data_analyser.core.exceptions import YAMLSyntaxException 4 | import pytest 5 | 6 | def test_assemble_pipe_ok() -> None: 7 | """ 8 | Test the assemble_pipe() method. 9 | It should runs whithout any problem and returns 10 | a GeometryConverter pipe. 11 | """ 12 | node_data = { 13 | 'test': 'oui', 14 | 'type': 'GeometryConverter', 15 | 'geometry_type': 'Node' 16 | } 17 | assert isinstance(PipeFactory.assemble_pipe(node_data, None), GeometryConverter) 18 | 19 | def test_assemble_pipe_no_type() -> None: 20 | """ 21 | Test the assemble_pipe() method. 22 | A YAML exception should be returned if the node_data 23 | given as parameter doesn't contain the 'type' key. 24 | """ 25 | node_data = { 26 | 'test': 'oui' 27 | } 28 | with pytest.raises(YAMLSyntaxException, match='Each node of the tree \(pipe\) should have a type defined.'): 29 | PipeFactory.assemble_pipe(node_data, None) 30 | 31 | def test_assemble_pipe_wrong_type() -> None: 32 | """ 33 | Test the assemble_pipe() method. 34 | A YAML exception should be returned if the node_data 35 | given as parameter contains a 'type' which doesn't exist. 36 | """ 37 | node_data = { 38 | 'test': 'oui', 39 | 'type': 'wrongtypewhichdoesntexist' 40 | } 41 | with pytest.raises(YAMLSyntaxException, match='The type wrongtypewhichdoesntexist doesn\'t exist.'): 42 | PipeFactory.assemble_pipe(node_data, None) 43 | -------------------------------------------------------------------------------- /tests/assembler/test_pipeline_assembler.py: -------------------------------------------------------------------------------- 1 | from nominatim_data_analyser.core.pipes.data_fetching.sql_processor import SQLProcessor 2 | from nominatim_data_analyser.core.pipes.output_formatters import GeoJSONFeatureConverter 3 | from nominatim_data_analyser.core.pipes.data_processing import GeometryConverter 4 | from nominatim_data_analyser.core.pipes.filling_pipe import FillingPipe 5 | from nominatim_data_analyser.core.assembler import PipelineAssembler 6 | from nominatim_data_analyser.core.pipe import Pipe 7 | import pytest 8 | 9 | def test_on_new_node(pipeline_assembler: PipelineAssembler, filling_pipe: Pipe) -> None: 10 | """ 11 | Test the on_new_node() method. 12 | The given node is not of type 'ROOT_NODE' so it should be 13 | created, plugged to the top pipe of the nodes history and added 14 | on the top of the nodes history deque. 15 | """ 16 | node = { 17 | 'type': 'GeometryConverter', 18 | 'geometry_type': 'Node' 19 | } 20 | #Add a FillingPipe as first pipe to the nodes history. 21 | first_pipe = filling_pipe 22 | pipeline_assembler.nodes_history.append(first_pipe) 23 | pipeline_assembler.on_new_node(node) 24 | new_top_node = pipeline_assembler.nodes_history.pop() 25 | #Check that the new top node is the right type. 26 | assert isinstance(new_top_node, GeometryConverter) 27 | #Check that the new top node is the one plugged to the first_pipe. 28 | assert first_pipe.next_pipes.pop() == new_top_node 29 | 30 | def test_on_new_node_root(pipeline_assembler: PipelineAssembler) -> None: 31 | """ 32 | Test the on_new_node() method. 33 | The given node is of type 'ROOT_NODE', hence a 34 | FillingPipe should be added to the nodes history as the first pipe. 35 | """ 36 | node = { 37 | 'type': 'ROOT_NODE' 38 | } 39 | assert len(pipeline_assembler.nodes_history) == 0 40 | pipeline_assembler.on_new_node(node) 41 | assert isinstance(pipeline_assembler.nodes_history.pop(), FillingPipe) 42 | 43 | def test_on_backtrack(pipeline_assembler: PipelineAssembler, 44 | filling_pipe: FillingPipe, 45 | geometry_converter: GeometryConverter, 46 | geojson_feature_converter: GeoJSONFeatureConverter, 47 | sql_processor: SQLProcessor) -> None: 48 | """ 49 | Test the on_backtrack() method. 50 | By applying two batracks the two last 51 | pipes of the nodes_history should be removed. 52 | """ 53 | pipeline_assembler.nodes_history.extend([ 54 | filling_pipe, 55 | geometry_converter, 56 | geojson_feature_converter, 57 | sql_processor 58 | ]) 59 | pipeline_assembler.on_backtrack() 60 | pipeline_assembler.on_backtrack() 61 | assert pipeline_assembler.nodes_history.pop() == geometry_converter 62 | 63 | def test_assemble() -> None: 64 | """ 65 | Test the assemble() method. 66 | A basic pipeline specification is given, only the execution is tested. 67 | """ 68 | pipeline_specification = { 69 | 'QUERY': { 70 | 'type': 'SQLProcessor', 71 | 'query': 'SELECT 1 FROM foo' 72 | } 73 | } 74 | pipeline_assembler = PipelineAssembler(pipeline_specification, 'test_rule') 75 | assert isinstance(pipeline_assembler.assemble(), FillingPipe) 76 | 77 | @pytest.fixture 78 | def pipeline_assembler() -> PipelineAssembler: 79 | return PipelineAssembler({}, 'test_rule') 80 | -------------------------------------------------------------------------------- /tests/config/test_config.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import yaml 3 | 4 | from nominatim_data_analyser.config import Config, load_config 5 | 6 | def test_load_default_config() -> None: 7 | """ 8 | Test the load_config() method. The default config should be 9 | returned because no config.yaml file is present in the 10 | default_config folder used as the config_folder_path. 11 | """ 12 | load_config(None) 13 | assert Config.values['Dsn'] == 'dbname=nominatim' 14 | assert Config.values['RulesFolderPath'] == 'qa-data' 15 | 16 | def test_load_custom_config(tmp_path) -> None: 17 | """ 18 | Test the load_config() method. The custom config should be 19 | returned because one config.yaml file is present in the 20 | custom_config folder used as the config_folder_path. 21 | """ 22 | cfgfile = tmp_path / 'myconfig.yaml' 23 | cfgfile.write_text("Dsn: 'custom_dsn'") 24 | 25 | load_config(cfgfile) 26 | 27 | assert Config.values['Dsn'] == 'custom_dsn' 28 | assert Config.values['RulesFolderPath'] == 'qa-data' 29 | 30 | def test_load_broken_config(tmp_path) -> None: 31 | """ 32 | Test the load_config() method. A YAMLError exception should 33 | be raised as the config file has a wrong syntax. 34 | """ 35 | cfgfile = tmp_path / 'myconfig.yaml' 36 | cfgfile.write_text(">>>>>>>>Dsn: 'custom_dsn'") 37 | 38 | with pytest.raises(yaml.YAMLError): 39 | load_config(cfgfile) 40 | -------------------------------------------------------------------------------- /tests/core/rules/rule1.yaml: -------------------------------------------------------------------------------- 1 | FILLING: 2 | type: FillingPipe 3 | out: 4 | GEOJSON: 5 | type: GeoJSONFormatter -------------------------------------------------------------------------------- /tests/core/rules/rule2.yaml: -------------------------------------------------------------------------------- 1 | FILLING: 2 | type: FillingPipe 3 | out: 4 | GEOJSON: 5 | type: GeoJSONFormatter -------------------------------------------------------------------------------- /tests/core/test_core.py: -------------------------------------------------------------------------------- 1 | 2 | from pathlib import Path 3 | 4 | import pytest 5 | from nominatim_data_analyser.config import Config 6 | from nominatim_data_analyser.core.core import Core 7 | 8 | class TestConfig: 9 | 10 | @pytest.fixture(autouse=True) 11 | def setup(self, tmp_path) -> None: 12 | self.rulepath = tmp_path / 'rules' 13 | cfgfile = tmp_path / 'custom_config.yaml' 14 | cfgfile.write_text(f"RulesFolderPath: '{self.rulepath}'") 15 | 16 | self.core = Core(config_file=cfgfile) 17 | self.core.rules_path = Path(__file__).parent / 'rules' 18 | 19 | def test_execute_one(self) -> None: 20 | """ 21 | Test the execute_one() method. The rule executed only generate a 22 | dumb geojson file. Therefore we only check if the file is created 23 | as expected. 24 | """ 25 | self.core.execute_one('rule1') 26 | assert (self.rulepath / 'rule1/geojson/rule1.json').is_file() 27 | 28 | def test_execute_all(self) -> None: 29 | """ 30 | Test the execute_all() method. The two rules only generate a 31 | dumb geojson file. Therefore we only check if the file are created 32 | as expected. 33 | """ 34 | self.core.execute_all() 35 | assert (self.rulepath / 'rule1/geojson/rule1.json').is_file() 36 | assert (self.rulepath / 'rule2/geojson/rule2.json').is_file() 37 | -------------------------------------------------------------------------------- /tests/dynamic_value/test_resolver.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from nominatim_data_analyser.core.dynamic_value import Variable 3 | from nominatim_data_analyser.core.dynamic_value.resolver import (_contains_dynamic_value, 4 | _is_dynamic_value, 5 | _resolve_if_resolvable, 6 | is_resolvable, resolve_all, 7 | resolve_one) 8 | 9 | def test_resolve_all() -> None: 10 | """ 11 | Test the resolve_all() function. Nested 12 | DynamicValue is tested here. 13 | """ 14 | variable1 = Variable('var_name1') 15 | variable2 = Variable('var_name2') 16 | variable3 = Variable('var_name3') 17 | data = { 18 | 'var_name1': variable2, 19 | 'var_name2': variable3, 20 | 'var_name3': 'var_value3', 21 | } 22 | assert resolve_all(variable1, data) == 'var_value3' 23 | 24 | def test_resolve_one(variable: Variable) -> None: 25 | """ 26 | Test the resolve_one() function with different 27 | possible data types. If any Variable is contained inside 28 | the data it gets resolved. 29 | """ 30 | data = {'var_name': 'var_value'} 31 | to_resolve = [ 32 | variable, 33 | [20, variable, 20, variable], 34 | (20, variable), 35 | {variable: 20, 'key': variable}, 36 | [20, 20], 37 | (20, 20), 38 | {20:20, 'key': 'value'}, 39 | 20 40 | ] 41 | expected = [ 42 | 'var_value', 43 | [20, 'var_value', 20, 'var_value'], 44 | (20, 'var_value'), 45 | {'var_value': 20, 'key': 'var_value'}, 46 | [20, 20], 47 | (20, 20), 48 | {20:20, 'key': 'value'}, 49 | 20 50 | ] 51 | for i in range(len(to_resolve)): 52 | assert resolve_one(to_resolve[i], data) == expected[i] 53 | 54 | def test_is_resolvable(variable: Variable) -> None: 55 | """ 56 | Test the _is_resolvable() function with different 57 | possible data types. If any Variable is contained inside 58 | the data, True should be returned. 59 | """ 60 | assert is_resolvable(variable) == True 61 | assert is_resolvable([20, variable, 20]) == True 62 | assert is_resolvable((20, variable, 20)) == True 63 | assert is_resolvable({variable: 20, 'key': 'value'}) == True 64 | assert is_resolvable({20: variable, 'key': 'value'}) == True 65 | assert is_resolvable({20: 20, 'key': 'value'}) == False 66 | assert is_resolvable((20, 20, 20)) == False 67 | assert is_resolvable([20, 20, 20]) == False 68 | assert is_resolvable(20) == False 69 | 70 | def test_contains_dynamic_value(variable: Variable) -> None: 71 | """ 72 | Test the _contains_dynamic_value() function with one 73 | array containing a dynamic value and another that does not. 74 | """ 75 | assert _contains_dynamic_value([20, variable, 20]) == True 76 | assert _contains_dynamic_value([20, 20, 20]) == False 77 | 78 | def test_is_dynamic_value(variable: Variable) -> None: 79 | """ 80 | Test the _is_dynamic_value() function with a 81 | dynamic value and with a classic value. 82 | """ 83 | assert _is_dynamic_value(variable) == True 84 | assert _is_dynamic_value(20) == False 85 | 86 | def test_resolve_if_resolvable(variable: Variable) -> None: 87 | """ 88 | Test the _resolve_if_resolvable() function with a 89 | resolvable and an unresolvable data. 90 | """ 91 | data = {'var_name': 'value'} 92 | assert _resolve_if_resolvable(data, variable) == 'value' 93 | assert _resolve_if_resolvable(data, 20) == 20 94 | 95 | @pytest.fixture 96 | def variable() -> Variable: 97 | return Variable('var_name') 98 | -------------------------------------------------------------------------------- /tests/dynamic_value/test_switch.py: -------------------------------------------------------------------------------- 1 | from nominatim_data_analyser.core.dynamic_value.switch import Switch 2 | import pytest 3 | 4 | def test_resolve_switch_ok(switch: Switch) -> None: 5 | """ 6 | Test the resolve() method of the Switch class without 7 | raising any exception. 8 | """ 9 | data = { 10 | 'test_key1': 'test_val1', 11 | 'test_expression': 'case2', 12 | 'test_key2': 'test_val3' 13 | } 14 | assert switch.resolve(data) == 'result2' 15 | 16 | def test_resolve_switch_expression_dont_exist(switch: Switch) -> None: 17 | """ 18 | Test the resolve() method of the Switch class by using a data 19 | dictionnary which doesn't contain the switch expression in its keys. 20 | """ 21 | data = { 22 | 'test_key1': 'test_val1', 23 | 'test_key2': 'case2', 24 | 'test_key2': 'test_val3' 25 | } 26 | with pytest.raises(Exception, match='The expression test_expression was not found in the input dictionnary.'): 27 | switch.resolve(data) 28 | 29 | def test_resolve_switch_case_dont_exist(switch: Switch) -> None: 30 | """ 31 | Test the resolve() method of the Switch class by using a 32 | case which isn't configured in the switch. 33 | """ 34 | data = { 35 | 'test_key1': 'test_val1', 36 | 'test_expression': 'wrong_case', 37 | 'test_key2': 'test_val3' 38 | } 39 | with pytest.raises(Exception, match=r'The case wrong_case is not in the configured switch cases.*'): 40 | switch.resolve(data) 41 | 42 | @pytest.fixture 43 | def switch() -> Switch: 44 | return Switch('test_expression', { 45 | 'case1': 'result1', 46 | 'case2': 'result2', 47 | 'case3': 'result3' 48 | }) 49 | -------------------------------------------------------------------------------- /tests/dynamic_value/test_variable.py: -------------------------------------------------------------------------------- 1 | 2 | from nominatim_data_analyser.core.dynamic_value import Variable 3 | import pytest 4 | 5 | def test_resolve_variable_ok(variable: Variable) -> None: 6 | """ 7 | Test the resolve() method of the Variable class without 8 | raising any exception. 9 | """ 10 | data = { 11 | 'test_key1': 'test_val1', 12 | 'var_name': 'test_val2', 13 | 'test_key2': 'test_val3' 14 | } 15 | assert variable.resolve(data) == 'test_val2' 16 | 17 | def test_resolve_variable_dont_exist(variable: Variable) -> None: 18 | """ 19 | Test the resolve() method of the Variable class by 20 | giving data without the var_name inside. 21 | """ 22 | data = { 23 | 'test_key1': 'test_val1', 24 | 'test_key2': 'test_val2', 25 | 'test_key3': 'test_val3' 26 | } 27 | with pytest.raises(Exception, match='The variable name var_name was not found in the input dictionary.'): 28 | variable.resolve(data) 29 | 30 | @pytest.fixture 31 | def variable() -> Variable: 32 | return Variable('var_name') 33 | 34 | -------------------------------------------------------------------------------- /tests/model/test_node.py: -------------------------------------------------------------------------------- 1 | from nominatim_data_analyser.core.model.node import Node 2 | from geojson.feature import Feature 3 | 4 | 5 | def test_create_from_WKT_string() -> None: 6 | node = Node.create_from_WKT_string('POINT(10 15)') 7 | assert node.coordinates[0] == 10 and node.coordinates[1] == 15 8 | 9 | def test_to_geojson_feature() -> None: 10 | """ 11 | Test the to_geojson_feature() method. 12 | The created feature should have the right geometry, id and properties. 13 | """ 14 | node = Node.create_from_WKT_string('POINT(10 15)') 15 | feature: Feature = node.to_geojson_feature(2, {'prop1': 'val1'}) 16 | assert feature['geometry']['type'] == 'Point' 17 | assert feature['geometry']['coordinates'] == [10, 15] 18 | assert feature['id'] == 2 19 | assert feature['properties'] == {'prop1': 'val1'} 20 | -------------------------------------------------------------------------------- /tests/pipes/output_formatters/test_geojson_feature_converter.py: -------------------------------------------------------------------------------- 1 | from nominatim_data_analyser.core.pipes.output_formatters import GeoJSONFeatureConverter 2 | from nominatim_data_analyser.core.model import Node 3 | from geojson.feature import Feature 4 | 5 | def test_on_created_geojson_feature_converter(geojson_feature_converter: GeoJSONFeatureConverter) -> None: 6 | """ 7 | Test the on_created() method of the GeoJSONFeatureConverter. 8 | """ 9 | geojson_feature_converter.properties_pattern = None 10 | geojson_feature_converter.data['properties'] = {'prop1': 'val1'} 11 | geojson_feature_converter.on_created() 12 | assert geojson_feature_converter.properties_pattern == {'prop1': 'val1'} 13 | 14 | def test_process_geojson_feature_converter(geojson_feature_converter: GeoJSONFeatureConverter) -> None: 15 | """ 16 | Test the process() method of the GeoJSONFeatureConverter. 17 | """ 18 | data = { 19 | 'geometry_holder': Node.create_from_WKT_string('POINT(10 15)') 20 | } 21 | feature: Feature = geojson_feature_converter.process(data) 22 | assert feature['geometry']['type'] == 'Point' 23 | assert feature['geometry']['coordinates'] == [10, 15] 24 | assert feature['properties'] == {'prop1': 'val1', 'prop2': 'val2'} 25 | -------------------------------------------------------------------------------- /tests/pipes/output_formatters/test_geojson_formatter.py: -------------------------------------------------------------------------------- 1 | from nominatim_data_analyser.core.pipes.output_formatters import GeoJSONFormatter 2 | from geojson import Feature, Point, FeatureCollection, loads 3 | from nominatim_data_analyser.config import Config 4 | 5 | def test_process_geojson_formatter(config: Config, 6 | geojson_formatter: GeoJSONFormatter, 7 | tmp_path) -> None: 8 | """ 9 | Test the process() method of the GeoJSONFormatter. 10 | A temporary folder is used as the base folder. 11 | 12 | the 'test_folder' doesn't exist initially and should be well 13 | created by the method. 14 | """ 15 | config.values['WebPrefixPath'] = 'test_prefix_path' 16 | geojson_formatter.file_name = 'test_file' 17 | geojson_formatter.base_folder_path = tmp_path / 'test_folder' 18 | 19 | features = [ 20 | Feature(geometry=Point((5, 2))), 21 | Feature(geometry=Point((4, 1))), 22 | Feature(geometry=Point((10, 20))) 23 | ] 24 | 25 | result: str = geojson_formatter.process(features) 26 | assert result == 'test_prefix_path/test_rule/geojson/test_file.json' 27 | 28 | #Verify the content of the geojson created 29 | with open(tmp_path/'test_folder/test_file.json' , 'r') as file: 30 | assert loads(file.read()) == FeatureCollection(features) 31 | -------------------------------------------------------------------------------- /tests/pipes/output_formatters/test_vector_tile_formatter.py: -------------------------------------------------------------------------------- 1 | from nominatim_data_analyser.core.pipes import VectorTileFormatter 2 | from nominatim_data_analyser.config import Config 3 | from geojson import Feature, Point 4 | 5 | def test_process_vector_tile_formatter(vector_tile_formatter: VectorTileFormatter, 6 | config: Config, 7 | tmp_path, 8 | monkeypatch) -> None: 9 | """ 10 | test the process() method. 11 | A temporary folder is used as the base folder. 12 | 13 | the 'test_folder' doesn't exist initially and should be well 14 | created by the method. 15 | 16 | The call to Tippecanoe is mocked to nothing because we dont want 17 | to test Tippecanoe. 18 | """ 19 | config.values['WebPrefixPath'] = 'test_prefix_path' 20 | vector_tile_formatter.base_folder_path = tmp_path / 'test_folder' 21 | 22 | features = [ 23 | Feature(geometry=Point((5, 2))), 24 | Feature(geometry=Point((4, 1))), 25 | Feature(geometry=Point((10, 20))) 26 | ] 27 | 28 | #Mock the call to Tippecanoe 29 | monkeypatch.setattr('nominatim_data_analyser.core.pipes.output_formatters.vector_tile_formatter.VectorTileFormatter.call_tippecanoe', 30 | lambda self, output_dir, feature_collection: None) 31 | 32 | web_path = vector_tile_formatter.process(features) 33 | assert web_path == 'test_prefix_path/test_rule/vector-tiles/{z}/{x}/{y}.pbf' 34 | -------------------------------------------------------------------------------- /tests/pipes/rules_specific_pipes/addr_house_number_no_digit/test_digits_filter.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from nominatim_data_analyser.core.pipes.rules_specific_pipes import \ 3 | AddrHouseNumberNoDigitFilter 4 | from nominatim_data_analyser.core.qa_rule import ExecutionContext 5 | 6 | 7 | def test_process_addr_HN_no_digit_filter(digits_filter: AddrHouseNumberNoDigitFilter) -> None: 8 | """ 9 | Test the process() method of the custom pipe AddrHouseNumberNoDigitFilter. 10 | Data where housenumber doesnt contain any number inside should be returned, otherwise 11 | the method should returns None. 12 | """ 13 | data_with_numbers = {'housenumber': '15aa2', 'dumbfield': 'dumbvalue'} 14 | data_without_numbers = {'housenumber': 'aa', 'dumbfield': 'dumbvalue'} 15 | 16 | assert digits_filter.process(data_with_numbers) == None 17 | assert digits_filter.process(data_without_numbers) == data_without_numbers 18 | 19 | @pytest.fixture 20 | def digits_filter(execution_context: ExecutionContext) -> AddrHouseNumberNoDigitFilter: 21 | return AddrHouseNumberNoDigitFilter({}, execution_context) 22 | -------------------------------------------------------------------------------- /tests/pipes/rules_specific_pipes/duplicate_label_role/test_duplicate_label_role_CFC.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from nominatim_data_analyser.core.model import Node 3 | from nominatim_data_analyser.core.pipes.rules_specific_pipes import \ 4 | DuplicateLabelRoleCustomFeatureConverter 5 | from nominatim_data_analyser.core.qa_rule import ExecutionContext 6 | from geojson.feature import Feature 7 | 8 | 9 | def test_process_duplicate_label_role_CFC(duplicate_label_role_CFC: DuplicateLabelRoleCustomFeatureConverter) -> None: 10 | """ 11 | Test the process() method of the custom pipe DuplicateLabelRoleCustomFeatureConverter. 12 | The method should be procuding a Feature with the expected_properties inside. 13 | """ 14 | data = { 15 | 'osm_id': 'dumb_osm_id', 16 | 'members': ['w8125151','outer','w249285853','inner', 17 | 'w25151','label','w24953','inner', 18 | 'w8121','label','w5853','label'], 19 | 'geometry_holder': Node.create_from_WKT_string('POINT(10 5)') 20 | } 21 | expected_properties = { 22 | 'relation_id': 'dumb_osm_id', 23 | 'w/@idLabel 1': '25151', 24 | 'w/@idLabel 2': '8121', 25 | 'w/@idLabel 3': '5853' 26 | } 27 | result = duplicate_label_role_CFC.process(data) 28 | assert isinstance(result, Feature) 29 | assert result['properties'] == expected_properties 30 | 31 | @pytest.fixture 32 | def duplicate_label_role_CFC(execution_context: ExecutionContext) -> DuplicateLabelRoleCustomFeatureConverter: 33 | return DuplicateLabelRoleCustomFeatureConverter({}, execution_context) 34 | -------------------------------------------------------------------------------- /tests/pipes/rules_specific_pipes/place_nodes_close/test_place_nodes_close_CFC.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from nominatim_data_analyser.core.model import Node 3 | from nominatim_data_analyser.core.pipes.rules_specific_pipes import \ 4 | PlaceNodesCloseCustomFeatureConverter 5 | from nominatim_data_analyser.core.qa_rule import ExecutionContext 6 | from geojson.feature import Feature 7 | 8 | def test_process_place_nodes_close_CFC(place_nodes_close_CFC: PlaceNodesCloseCustomFeatureConverter) -> None: 9 | """ 10 | Test the process() method of the custom pipe PlaceNodesCloseCustomFeatureConverter. 11 | The method should be procuding a Feature with the expected_properties inside. 12 | """ 13 | data = { 14 | 'osm_id': 'dumb_osm_id', 15 | 'common_ids': ['CID1', 'CID2', 'CID3'], 16 | 'geometry_holder': Node.create_from_WKT_string('POINT(10 5)') 17 | } 18 | expected_properties = { 19 | 'node_id': 'dumb_osm_id', 20 | 'n/@idClose node 1': 'CID1', 21 | 'n/@idClose node 2': 'CID2', 22 | 'n/@idClose node 3': 'CID3' 23 | } 24 | result = place_nodes_close_CFC.process(data) 25 | assert isinstance(result, Feature) 26 | assert result['properties'] == expected_properties 27 | 28 | @pytest.fixture 29 | def place_nodes_close_CFC(execution_context: ExecutionContext) -> PlaceNodesCloseCustomFeatureConverter: 30 | return PlaceNodesCloseCustomFeatureConverter({}, execution_context) 31 | -------------------------------------------------------------------------------- /tests/pipes/rules_specific_pipes/same_wikidata/test_same_wikidata_CFC.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from nominatim_data_analyser.core.pipes.rules_specific_pipes import \ 3 | SameWikiDataFeatureConverter 4 | from nominatim_data_analyser.core.qa_rule import ExecutionContext 5 | 6 | 7 | def test_same_wikidata_CFC(same_wikidata_CFC: SameWikiDataFeatureConverter) -> None: 8 | """ 9 | Test the process() method of the custom pipe SameWikiDataFeatureConverter. 10 | The method should be procuding a List of features with the right values 11 | matching the expected_results. 12 | """ 13 | data = [ 14 | { 15 | 'wikidata': 'dumb_wikidata1', 16 | 'ids': ['ID1', 'ID2', 'ID3'], 17 | 'centroids': ['POINT(5 2)', 'POINT(8 3)', None] 18 | }, 19 | { 20 | 'wikidata': 'dumb_wikidata2', 21 | 'ids': ['ID4', 'ID5'], 22 | 'centroids': ['POINT(8 10)', 'POINT(80 30)'] 23 | } 24 | ] 25 | expected_results = [ 26 | { 27 | "geometry": {"coordinates": [5.0, 2.0], "type": "Point"}, 28 | "id": 0, 29 | "properties": {"n/@idNode in common 1": "ID2", "n/@idNode in common 2": "ID3", "node_id": "ID1", "wikidata in common": "dumb_wikidata1"}, 30 | "type": "Feature" 31 | }, 32 | { 33 | "geometry": {"coordinates": [8.0, 3.0], "type": "Point"}, 34 | "id": 1, 35 | "properties": {"n/@idNode in common 1": "ID1", "n/@idNode in common 2": "ID3", "node_id": "ID2", "wikidata in common": "dumb_wikidata1"}, 36 | "type": "Feature" 37 | }, 38 | { 39 | "geometry": {"coordinates": [8.0, 10.0], "type": "Point"}, 40 | "id": 2, 41 | "properties": {"n/@idNode in common 1": "ID5", "node_id": "ID4", "wikidata in common": "dumb_wikidata2"}, 42 | "type": "Feature"}, 43 | { 44 | "geometry": {"coordinates": [80.0, 30.0], "type": "Point"}, 45 | "id": 3, 46 | "properties": {"n/@idNode in common 1": "ID4", "node_id": "ID5", "wikidata in common": "dumb_wikidata2"}, 47 | "type": "Feature" 48 | } 49 | ] 50 | results = same_wikidata_CFC.process(data) 51 | assert results == expected_results 52 | 53 | @pytest.fixture 54 | def same_wikidata_CFC(execution_context: ExecutionContext) -> SameWikiDataFeatureConverter: 55 | return SameWikiDataFeatureConverter({}, execution_context) 56 | -------------------------------------------------------------------------------- /tests/pipes/test_geometry_converter.py: -------------------------------------------------------------------------------- 1 | from nominatim_data_analyser.core.pipes.data_processing import GeometryConverter 2 | 3 | def test_on_created_geometry_converter(geometry_converter: GeometryConverter) -> None: 4 | """ 5 | Test the on_created() method of the GeometryConverter. 6 | """ 7 | geometry_converter.geometry_type = None 8 | geometry_converter.data['geometry_type'] = 'Node' 9 | geometry_converter.on_created() 10 | assert geometry_converter.geometry_type == 'Node' 11 | 12 | def test_process_geometry_converter(geometry_converter: GeometryConverter) -> None: 13 | """ 14 | Test the process() method of the GeometryConverter. 15 | """ 16 | data_result = geometry_converter.process({'geometry_holder': 'POINT(10 15)'}) 17 | assert data_result['geometry_holder'].coordinates[0] == 10 and data_result['geometry_holder'].coordinates[1] == 15 18 | 19 | def test_process_geometry_converter_none_geometry_holder(geometry_converter: GeometryConverter) -> None: 20 | """ 21 | Test the process() method of the GeometryConverter with one data containing a None value for the geometry_holder. 22 | """ 23 | data_result = geometry_converter.process({'geometry_holder': None}) 24 | assert data_result is None 25 | -------------------------------------------------------------------------------- /tests/pipes/test_pipe.py: -------------------------------------------------------------------------------- 1 | 2 | from nominatim_data_analyser.core.exceptions.yaml_syntax_exception import YAMLSyntaxException 3 | from nominatim_data_analyser.core.pipes.data_processing import GeometryConverter 4 | from nominatim_data_analyser.core.qa_rule import ExecutionContext 5 | from nominatim_data_analyser.core.pipes import FillingPipe 6 | import logging 7 | import pytest 8 | 9 | def test_plug_pipe(filling_pipe: FillingPipe, geometry_converter: GeometryConverter) -> None: 10 | filling_pipe.plug_pipe(geometry_converter) 11 | assert filling_pipe.next_pipes.pop() == geometry_converter 12 | 13 | def test_process_and_next(filling_pipe: FillingPipe, execution_context: ExecutionContext, monkeypatch) -> None: 14 | """ 15 | Test the process_and_next() method. The process() method of the 16 | two pipes should be called. 17 | """ 18 | filling_pipe2 = FillingPipe({}, execution_context) 19 | filling_pipe.next_pipes.add(filling_pipe2) 20 | x = 0 21 | def callback(self, data = None): 22 | nonlocal x 23 | x += 1 24 | monkeypatch.setattr('nominatim_data_analyser.core.pipes.filling_pipe.FillingPipe.process', 25 | callback) 26 | filling_pipe.process_and_next() 27 | assert x == 2 28 | 29 | def test__str__(filling_pipe: FillingPipe) -> None: 30 | """ 31 | Test the __str__ 32 | """ 33 | assert str(filling_pipe) == type(filling_pipe).__name__ + ' ' + str(filling_pipe.id) 34 | 35 | def test_extract_data_basic(filling_pipe: FillingPipe) -> None: 36 | """ 37 | Test the extract_data() method with no default value provided and 38 | required = False. 39 | """ 40 | filling_pipe.data['test_data'] = 'test_data_value' 41 | assert filling_pipe.extract_data('test_data') == 'test_data_value' 42 | #The second time the value is None as it has already been extracted (default = None is returned). 43 | assert filling_pipe.extract_data('test_data') == None 44 | 45 | def test_extract_data_with_default(filling_pipe: FillingPipe) -> None: 46 | """ 47 | Test the extract_data() method with a default value provided and 48 | required = False. 49 | """ 50 | assert filling_pipe.extract_data('test_data', 'test_data_value') == 'test_data_value' 51 | 52 | @pytest.mark.parametrize("default", [(None), ('test_data_value')]) 53 | def test_extract_data_with_required(filling_pipe: FillingPipe, default) -> None: 54 | """ 55 | Test the extract_data() method with/without a default value provided and 56 | required = True. 57 | """ 58 | with pytest.raises(YAMLSyntaxException, match='The field "test_data" is required for the pipe of type FillingPipe'): 59 | filling_pipe.extract_data('test_data', default, True) 60 | 61 | def test_log(filling_pipe: FillingPipe) -> None: 62 | """ 63 | Test the execution of the log() method. 64 | """ 65 | filling_pipe.log('test_message', logging.WARN) 66 | 67 | @pytest.fixture 68 | def filling_pipe(execution_context: ExecutionContext) -> FillingPipe: 69 | return FillingPipe({}, execution_context) 70 | -------------------------------------------------------------------------------- /tests/pipes/test_sql_processor.py: -------------------------------------------------------------------------------- 1 | from nominatim_data_analyser.core.pipes.data_fetching import SQLProcessor 2 | from nominatim_data_analyser.config import Config 3 | 4 | 5 | def test_on_created_sql_processor(sql_processor: SQLProcessor): 6 | """ 7 | Test the on_created() method of the SQLProcessor. 8 | """ 9 | sql_processor.query = None 10 | sql_processor.data['query'] = 'QUERY' 11 | sql_processor.on_created() 12 | assert sql_processor.query == 'QUERY' 13 | 14 | def test_execute_query(sql_processor: SQLProcessor, dsn, temp_db_cursor): 15 | """ 16 | Test the execute_query() method of the SQLProcessor. 17 | """ 18 | temp_db_cursor.execute('CREATE TABLE test_table(val VARCHAR(255));') 19 | temp_db_cursor.execute(""" 20 | INSERT INTO test_table (val) VALUES ('test1'), ('test2'), ('test3'); 21 | """) 22 | sql_processor.query = 'SELECT * FROM test_table' 23 | 24 | Config.values['Dsn'] = dsn 25 | results = sql_processor.process(None) 26 | assert len(results) == 3 and results[0]['val'] == 'test1' and results[1]['val'] == 'test2' and results[2]['val'] == 'test3' 27 | -------------------------------------------------------------------------------- /tests/yaml_logic/test_yaml_loader.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import pytest 4 | import yaml 5 | 6 | from nominatim_data_analyser.core.dynamic_value.variable import Variable 7 | from nominatim_data_analyser.core.dynamic_value.switch import Switch 8 | import nominatim_data_analyser.core.yaml_logic.yaml_loader as yaml_loader 9 | from nominatim_data_analyser.core.yaml_logic.yaml_loader import load_yaml_rule 10 | from nominatim_data_analyser.core import Pipe 11 | 12 | def test_load_yaml_rule(yaml_path) -> None: 13 | """ 14 | Test the load_yaml_rule() function with a test yaml file. 15 | """ 16 | loaded_data = load_yaml_rule(yaml_path / 'test_load_yaml.yaml') 17 | assert loaded_data == { 18 | 'QUERY': { 19 | 'type': 'SQLProcessor', 20 | 'query': 'QUERY', 21 | 'out': { 22 | 'DUMB_PIPE': { 23 | 'type': 'DumbPipe' 24 | } 25 | } 26 | } 27 | } 28 | 29 | def test_load_wrong_yaml(yaml_path) -> None: 30 | """ 31 | Test the load_yaml_rule() function with a test yaml file which 32 | contains wrong syntax. A YAMLError should be raised while loading. 33 | """ 34 | with pytest.raises(yaml.YAMLError): 35 | load_yaml_rule(yaml_path / 'test_load_wrong_yaml.yaml') 36 | 37 | def test_construct_sub_pipeline(yaml_path) -> None: 38 | """ 39 | Test that the sub_pipeline_constructor() is well called when 40 | a specific type !sub-pipeline is present in the YAML file. 41 | 42 | The value should be a Pipe after loading. 43 | """ 44 | loaded_data = load_yaml_rule(yaml_path / 'test_construct_sub_pipeline.yaml') 45 | assert isinstance(loaded_data['QUERY']['sub_pipeline'], Pipe) 46 | 47 | def test_construct_switch(yaml_path) -> None: 48 | """ 49 | Test that the switch_constructor() is well called when 50 | a specific type !switch is present in the YAML file. 51 | 52 | The value should be a Switch after loading. 53 | """ 54 | loaded_data = load_yaml_rule(yaml_path / 'test_construct_switch.yaml') 55 | expected_cases = { 56 | 'case1': 'val1', 57 | 'case2': 'val2', 58 | 'case3': 'val3' 59 | } 60 | assert isinstance(loaded_data['DUMB_NODE']['value'], Switch) 61 | assert loaded_data['DUMB_NODE']['value'].expression == 'expression_value' 62 | assert loaded_data['DUMB_NODE']['value'].cases == expected_cases 63 | 64 | def test_construct_variable(yaml_path) -> None: 65 | """ 66 | Test that the variable_constructor() is well called when 67 | a specific type !variable is present in the YAML file. 68 | 69 | The value should be a Variable after loading. 70 | """ 71 | loaded_data = load_yaml_rule(yaml_path / 'test_construct_variable.yaml') 72 | assert isinstance(loaded_data['DUMB_NODE']['value'], Variable) 73 | assert loaded_data['DUMB_NODE']['value'].name == 'variable_name' 74 | 75 | @pytest.fixture 76 | def yaml_path() -> Path: 77 | return Path(__file__).parent / 'yaml' 78 | -------------------------------------------------------------------------------- /tests/yaml_logic/yaml/test_construct_sub_pipeline.yaml: -------------------------------------------------------------------------------- 1 | QUERY: 2 | type: SQLProcessor 3 | query: QUERY 4 | sub_pipeline: !sub-pipeline 5 | SUB_SQL_PROCESSOR: 6 | type: SQLProcessor 7 | query: QUERY 8 | out: 9 | FILLING_PIPE: 10 | type: FillingPipe 11 | out: 12 | DUMB_PIPE: 13 | type: DumbPipe -------------------------------------------------------------------------------- /tests/yaml_logic/yaml/test_construct_switch.yaml: -------------------------------------------------------------------------------- 1 | DUMB_NODE: 2 | value: !switch 3 | expression: expression_value 4 | cases: 5 | 'case1': val1 6 | 'case2': val2 7 | 'case3': val3 -------------------------------------------------------------------------------- /tests/yaml_logic/yaml/test_construct_variable.yaml: -------------------------------------------------------------------------------- 1 | DUMB_NODE: 2 | value: !variable variable_name -------------------------------------------------------------------------------- /tests/yaml_logic/yaml/test_load_wrong_yaml.yaml: -------------------------------------------------------------------------------- 1 | QUERY: 2 | type: SQLProcessor 3 | query: QUERY 4 | out: 5 | >>>>>DUMB_PIPE: 6 | type: DumbPipe -------------------------------------------------------------------------------- /tests/yaml_logic/yaml/test_load_yaml.yaml: -------------------------------------------------------------------------------- 1 | QUERY: 2 | type: SQLProcessor 3 | query: QUERY 4 | out: 5 | DUMB_PIPE: 6 | type: DumbPipe --------------------------------------------------------------------------------