├── website ├── .gitignore ├── .pipignore ├── Makefile ├── photon │ └── static │ │ ├── img │ │ ├── photon_logo.png │ │ ├── favicon_photon_logo.ico │ │ └── opensource.svg │ │ ├── font │ │ ├── droidsans-webfont.eot │ │ ├── droidsans-webfont.ttf │ │ ├── droidsans-webfont.woff │ │ ├── droidsans-bold-webfont.eot │ │ ├── droidsans-bold-webfont.ttf │ │ └── droidsans-bold-webfont.woff │ │ └── js │ │ ├── app.js │ │ ├── controllers.js │ │ └── angular-sanitize.js ├── config.js.example ├── Readme.md └── import_bano.py ├── settings.gradle ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── .gitmodules ├── src ├── main │ ├── java │ │ └── de │ │ │ └── komoot │ │ │ └── photon │ │ │ ├── query │ │ │ ├── RequestFactory.java │ │ │ ├── SimpleSearchRequest.java │ │ │ ├── BadRequestException.java │ │ │ ├── SimpleSearchRequestFactory.java │ │ │ ├── ReverseRequest.java │ │ │ ├── ReverseRequestFactory.java │ │ │ ├── SearchRequestBase.java │ │ │ ├── SearchRequestFactoryBase.java │ │ │ ├── BoundingBoxParamConverter.java │ │ │ ├── StructuredSearchRequestFactory.java │ │ │ ├── StructuredSearchRequest.java │ │ │ └── RequestBase.java │ │ │ ├── Updater.java │ │ │ ├── opensearch │ │ │ ├── PhotonIndex.java │ │ │ ├── ReverseQueryBuilder.java │ │ │ ├── CategoryFilter.java │ │ │ ├── NameCollector.java │ │ │ ├── BaseQueryBuilder.java │ │ │ ├── OpenSearchResult.java │ │ │ ├── OpenSearchReverseHandler.java │ │ │ ├── OpenSearchSearchHandler.java │ │ │ ├── OsmTagFilter.java │ │ │ ├── Updater.java │ │ │ ├── Importer.java │ │ │ ├── OpenSearchResultDeserializer.java │ │ │ └── OpenSearchStructuredSearchHandler.java │ │ │ ├── json │ │ │ ├── NominatimDumpFileFeatures.java │ │ │ ├── AddressLine.java │ │ │ ├── CountryInfo.java │ │ │ ├── DumpFields.java │ │ │ └── NominatimDumpHeader.java │ │ │ ├── UsageException.java │ │ │ ├── searcher │ │ │ ├── SearchHandler.java │ │ │ ├── TagFilterKind.java │ │ │ ├── ResultFormatter.java │ │ │ ├── PhotonResult.java │ │ │ ├── StreetDupesRemover.java │ │ │ ├── TagFilter.java │ │ │ └── GeocodeJsonFormatter.java │ │ │ ├── Importer.java │ │ │ ├── utils │ │ │ └── CorsMutuallyExclusiveValidator.java │ │ │ ├── nominatim │ │ │ ├── model │ │ │ │ ├── UpdateRow.java │ │ │ │ ├── ContextMap.java │ │ │ │ ├── NameMap.java │ │ │ │ ├── OsmlineRowMapper.java │ │ │ │ ├── AddressRow.java │ │ │ │ ├── AddressType.java │ │ │ │ ├── PlaceRowMapper.java │ │ │ │ └── NominatimAddressCache.java │ │ │ ├── DBDataAdapter.java │ │ │ ├── PostgisDataAdapter.java │ │ │ ├── NominatimConnector.java │ │ │ └── ImportThread.java │ │ │ ├── ConfigSynonyms.java │ │ │ ├── ConfigClassificationTerm.java │ │ │ ├── Constants.java │ │ │ ├── PhotonDocInterpolationSet.java │ │ │ ├── GenericSearchHandler.java │ │ │ ├── StatusRequestHandler.java │ │ │ ├── ConfigExtraTags.java │ │ │ └── metrics │ │ │ └── MetricsConfig.java │ └── resources │ │ └── log4j2.xml └── test │ ├── java │ └── de │ │ └── komoot │ │ └── photon │ │ ├── nominatim │ │ ├── model │ │ │ ├── AddressTypeTest.java │ │ │ ├── NameMapTest.java │ │ │ └── ContextMapTest.java │ │ └── testdb │ │ │ ├── Helpers.java │ │ │ ├── PhotonUpdateRow.java │ │ │ ├── CollectingUpdater.java │ │ │ ├── H2DataAdapter.java │ │ │ ├── CollectingImporter.java │ │ │ └── OsmlineTestRow.java │ │ ├── ServerDatabasePropertiesTest.java │ │ ├── ReflectionTestUtil.java │ │ ├── PhotonDocTest.java │ │ ├── metrics │ │ └── MetricsConfigTest.java │ │ ├── searcher │ │ ├── StreetDupesRemoverTest.java │ │ ├── TagFilterTest.java │ │ ├── MockPhotonResult.java │ │ └── GeocodeJsonFormatterTest.java │ │ ├── DatabasePropertiesTest.java │ │ ├── api │ │ ├── ApiBaseTester.java │ │ ├── ApiLanguagesTest.java │ │ ├── ApiMetricsTest.java │ │ └── ApiDedupeTest.java │ │ ├── query │ │ ├── QueryGeometryTest.java │ │ ├── QueryReverseTest.java │ │ ├── QueryFilterLayerTest.java │ │ └── QueryByLanguageTest.java │ │ ├── PhotonDocInterpolationSetTest.java │ │ ├── ESBaseTester.java │ │ └── TestServer.java │ └── resources │ └── test-schema.sql ├── .gitignore ├── .github ├── ISSUE_TEMPLATE.md └── CONTRIBUTING.md ├── continuously_update_from_nominatim.sh ├── docs ├── structured.md ├── categories.md └── synonyms.md └── gradlew.bat /website/.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'photon' 2 | 3 | -------------------------------------------------------------------------------- /website/.pipignore: -------------------------------------------------------------------------------- 1 | distribute 2 | ipython 3 | pip-tools 4 | ipdb -------------------------------------------------------------------------------- /website/Makefile: -------------------------------------------------------------------------------- 1 | serve: 2 | cd photon; python3 -m http.server 5001 3 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/komoot/photon/HEAD/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /website/photon/static/img/photon_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/komoot/photon/HEAD/website/photon/static/img/photon_logo.png -------------------------------------------------------------------------------- /website/photon/static/font/droidsans-webfont.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/komoot/photon/HEAD/website/photon/static/font/droidsans-webfont.eot -------------------------------------------------------------------------------- /website/photon/static/font/droidsans-webfont.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/komoot/photon/HEAD/website/photon/static/font/droidsans-webfont.ttf -------------------------------------------------------------------------------- /website/photon/static/font/droidsans-webfont.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/komoot/photon/HEAD/website/photon/static/font/droidsans-webfont.woff -------------------------------------------------------------------------------- /website/photon/static/img/favicon_photon_logo.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/komoot/photon/HEAD/website/photon/static/img/favicon_photon_logo.ico -------------------------------------------------------------------------------- /website/photon/static/font/droidsans-bold-webfont.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/komoot/photon/HEAD/website/photon/static/font/droidsans-bold-webfont.eot -------------------------------------------------------------------------------- /website/photon/static/font/droidsans-bold-webfont.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/komoot/photon/HEAD/website/photon/static/font/droidsans-bold-webfont.ttf -------------------------------------------------------------------------------- /website/photon/static/font/droidsans-bold-webfont.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/komoot/photon/HEAD/website/photon/static/font/droidsans-bold-webfont.woff -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "website/photon/static/leaflet.photon"] 2 | path = website/photon/static/leaflet.photon 3 | url = https://github.com/komoot/leaflet.photon.git 4 | -------------------------------------------------------------------------------- /website/config.js.example: -------------------------------------------------------------------------------- 1 | API_URL = 'http://localhost:5001/api/?'; 2 | TILELAYER = 'https://tile.openstreetmap.org/{z}/{x}/{y}.png'; 3 | CENTER = [52.3879, 13.0582]; 4 | MAXZOOM = 18; 5 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/query/RequestFactory.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.query; 2 | 3 | import io.javalin.http.Context; 4 | 5 | public interface RequestFactory { 6 | 7 | T create(Context context); 8 | } 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | photon_data/ 3 | dependency-reduced-pom.xml 4 | .gradle/ 5 | build 6 | 7 | .idea/ 8 | *.iml 9 | *.~* 10 | 11 | website/photon/config.js 12 | 13 | nb-configuration.xml 14 | 15 | nbactions.xml 16 | 17 | .classpath 18 | .project 19 | .settings/ 20 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-9.1.0-bin.zip 4 | networkTimeout=10000 5 | validateDistributionUrl=true 6 | zipStoreBase=GRADLE_USER_HOME 7 | zipStorePath=wrapper/dists 8 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/Updater.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon; 2 | 3 | /** 4 | * Interface for classes accepting database updates. 5 | */ 6 | public interface Updater { 7 | void addOrUpdate(Iterable docs); 8 | 9 | void delete(long docId); 10 | 11 | void finish(); 12 | } 13 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/opensearch/PhotonIndex.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.opensearch; 2 | 3 | public class PhotonIndex { 4 | public static final String NAME = "photon"; 5 | public static final String META_DB_PROPERTIES = "PhotonProperties"; 6 | 7 | private PhotonIndex() { 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/json/NominatimDumpFileFeatures.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.json; 2 | 3 | import com.fasterxml.jackson.annotation.JsonProperty; 4 | 5 | public class NominatimDumpFileFeatures { 6 | 7 | @JsonProperty("sorted_by_country") 8 | public boolean isSortedByCountry = true; 9 | 10 | @JsonProperty("has_addresslines") 11 | public boolean hasAddressLines = false; 12 | } 13 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/UsageException.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon; 2 | 3 | /** 4 | * Exception caused by some error from the user side. 5 | * 6 | * This exception type will be caught and the error printed without giving 7 | * the user a backtrace. 8 | */ 9 | public class UsageException extends RuntimeException { 10 | 11 | public UsageException(String message) { 12 | super(message); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/searcher/SearchHandler.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.searcher; 2 | 3 | import de.komoot.photon.query.RequestBase; 4 | 5 | import java.util.List; 6 | 7 | /** 8 | * Interface for a handler of search geocoding requests. 9 | */ 10 | public interface SearchHandler { 11 | 12 | List search(T searchRequest); 13 | 14 | String dumpQuery(T searchRequest); 15 | } 16 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/query/SimpleSearchRequest.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.query; 2 | 3 | /** 4 | * Collection of query parameters for a search request. 5 | */ 6 | public class SimpleSearchRequest extends SearchRequestBase { 7 | private String query; 8 | 9 | public String getQuery() { 10 | return query; 11 | } 12 | 13 | public void setQuery(String query) { 14 | this.query = query; 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/searcher/TagFilterKind.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.searcher; 2 | 3 | /** 4 | * List of filter types. 5 | */ 6 | public enum TagFilterKind { 7 | /// Include an object if it matches the filter parameters. 8 | INCLUDE, 9 | // Do not include an object if it matches the given filter parameters. 10 | EXCLUDE, 11 | // Do not include an object when it matches the value parameter. 12 | EXCLUDE_VALUE; 13 | } 14 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/json/AddressLine.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.json; 2 | 3 | import com.fasterxml.jackson.annotation.JsonProperty; 4 | 5 | public class AddressLine { 6 | 7 | @JsonProperty("place_id") 8 | public long placeId; 9 | 10 | @JsonProperty("rank_address") 11 | public int rankAddress; 12 | 13 | @JsonProperty("fromarea") 14 | public boolean fromArea; 15 | 16 | @JsonProperty("isaddress") 17 | public boolean isAddress; 18 | } 19 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/searcher/ResultFormatter.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.searcher; 2 | 3 | import java.io.IOException; 4 | import java.util.List; 5 | 6 | /** 7 | * Convert a list of results into an output string. 8 | */ 9 | public interface ResultFormatter { 10 | 11 | String convert(List results, String language, 12 | boolean withGeometry, boolean withDebugInfo, String queryDebugInfo) throws IOException; 13 | 14 | String formatError(String msg); 15 | } 16 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/query/BadRequestException.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.query; 2 | 3 | /** 4 | * Exception thrown when parsing the input request failed. 5 | */ 6 | public class BadRequestException extends RuntimeException { 7 | private final int httpStatus; 8 | 9 | public BadRequestException(int httpStatusCode, String message) { 10 | super(message); 11 | this.httpStatus = httpStatusCode; 12 | } 13 | 14 | public int getHttpStatus() { 15 | return httpStatus; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/test/java/de/komoot/photon/nominatim/model/AddressTypeTest.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.nominatim.model; 2 | 3 | import org.junit.jupiter.api.Test; 4 | 5 | import static org.junit.jupiter.api.Assertions.*; 6 | 7 | class AddressTypeTest { 8 | 9 | /** 10 | * All ranks covered by Nominatim must return a corresponding Photon rank. 11 | */ 12 | @Test 13 | void testAllRanksAreCovered() { 14 | for (int i = 4; i <= 30; ++i) { 15 | assertNotNull(AddressType.fromRank(i)); 16 | } 17 | } 18 | } -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/Importer.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon; 2 | 3 | /** 4 | * Interface for bulk imports from a data source like nominatim 5 | */ 6 | public interface Importer { 7 | /** 8 | * Add a new set of document to the Photon database. 9 | * 10 | * The document set must have been created from the same base document 11 | * and each document must have the same place ID. 12 | */ 13 | public void add(Iterable docs); 14 | 15 | /** 16 | * Finish up the import. 17 | */ 18 | public void finish(); 19 | } 20 | -------------------------------------------------------------------------------- /src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /website/Readme.md: -------------------------------------------------------------------------------- 1 | ## Website for photon.komoot.io 2 | 3 | This directory contains the website for photon's home page. 4 | 5 | ### Configuration 6 | 7 | Copy the example configuration into the photon directory: 8 | 9 | ``` 10 | cp config.js.example photon/config.js 11 | ``` 12 | 13 | Then adapt `config.js` to your needs. 14 | 15 | ### Running 16 | 17 | The website can be directly opened in your browser or served with any 18 | webserver that can serve static files. 19 | 20 | If you have Python3 installed, you can run: 21 | 22 | ``` 23 | make serve 24 | ``` 25 | 26 | Then go to http://localhost:5001/ and test it! 27 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | Submit a new issue only if you are sure it is a missing feature or a bug. Otherwise use the [discussion section](https://github.com/komoot/photon/discussions). 2 | 3 | The best way to get help about a Photon issue is to create a valid and detailed issue content, preferable with a unit or integration test reproducing the issue. Please include the version of Photon, the JVM and the operating system are you using and in case of a geocoding problem include a link to [komoot's photon instance](https://photon.komoot.io/). Also tell us your expected geocoding result. 4 | 5 | **Do not post screenshots** of commandline output or search results. Please copy the output into the issue instead. 6 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/utils/CorsMutuallyExclusiveValidator.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.utils; 2 | 3 | import com.beust.jcommander.IParametersValidator; 4 | import com.beust.jcommander.ParameterException; 5 | 6 | import java.util.Map; 7 | 8 | import static java.lang.Boolean.TRUE; 9 | 10 | public class CorsMutuallyExclusiveValidator implements IParametersValidator { 11 | @Override 12 | public void validate(Map parameters) throws ParameterException { 13 | if (parameters.get("-cors-any") == TRUE && parameters.get("-cors-origin") != null) { 14 | throw new ParameterException("Use only one cors configuration type."); 15 | } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/test/java/de/komoot/photon/nominatim/testdb/Helpers.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.nominatim.testdb; 2 | 3 | import org.locationtech.jts.geom.Geometry; 4 | import org.springframework.lang.Nullable; 5 | 6 | import java.sql.ResultSet; 7 | import java.sql.SQLException; 8 | 9 | /** 10 | * Helper functions used by H2 test database. 11 | *

12 | * See test-schema.sql for how they are used. 13 | */ 14 | public class Helpers { 15 | 16 | public static Geometry envelope(Geometry geom) { 17 | if (geom == null) 18 | return null; 19 | 20 | return geom.getEnvelope(); 21 | } 22 | 23 | @Nullable 24 | public static T extractGeometry(ResultSet rs, String columnName) throws SQLException { 25 | return (T) rs.getObject(columnName); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/nominatim/model/UpdateRow.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.nominatim.model; 2 | 3 | import java.util.Date; 4 | 5 | /** 6 | * Information about places in the database that need updating. 7 | */ 8 | public class UpdateRow { 9 | 10 | private Long placeId; 11 | private boolean toDelete; 12 | private Date updateDate; 13 | 14 | public UpdateRow(Long placeId, boolean toDelete, Date updateDate) { 15 | this.placeId = placeId; 16 | this.toDelete = toDelete; 17 | this.updateDate = updateDate; 18 | } 19 | 20 | public Long getPlaceId() { 21 | return placeId; 22 | } 23 | 24 | public boolean isToDelete() { 25 | return toDelete; 26 | } 27 | 28 | public Date getUpdateDate() { 29 | return updateDate; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/json/CountryInfo.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.json; 2 | 3 | import com.fasterxml.jackson.annotation.JsonProperty; 4 | 5 | import java.util.Map; 6 | 7 | public class CountryInfo { 8 | public static final String DOCUMENT_TYPE = "CountryInfo"; 9 | 10 | private String countryCode; 11 | 12 | private Map name = Map.of(); 13 | 14 | @JsonProperty("name") 15 | public void setName(Map names) { 16 | this.name = names; 17 | } 18 | 19 | @JsonProperty("country_code") 20 | public void setCountryCode(String countryCode) { 21 | this.countryCode = countryCode; 22 | } 23 | 24 | public Map getName() { 25 | return name; 26 | } 27 | 28 | public String getCountryCode() { 29 | return countryCode; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/test/java/de/komoot/photon/nominatim/testdb/PhotonUpdateRow.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.nominatim.testdb; 2 | 3 | import org.springframework.jdbc.core.JdbcTemplate; 4 | 5 | import java.util.Date; 6 | 7 | public class PhotonUpdateRow { 8 | private String rel; 9 | private Long placeId; 10 | private String operation; 11 | 12 | 13 | public PhotonUpdateRow(String rel, Long placeId, String operation) { 14 | this.rel = rel; 15 | this.placeId = placeId; 16 | this.operation = operation; 17 | } 18 | 19 | public PhotonUpdateRow add(JdbcTemplate jdbc) { 20 | jdbc.update("INSERT INTO photon_updates (rel, place_id, operation, indexed_date)" 21 | + "VALUES (?, ?, ?, ?)", 22 | rel, placeId, operation, new Date()); 23 | 24 | return this; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/opensearch/ReverseQueryBuilder.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.opensearch; 2 | 3 | import org.locationtech.jts.geom.Point; 4 | 5 | public class ReverseQueryBuilder extends BaseQueryBuilder { 6 | 7 | public ReverseQueryBuilder(Point location, double radius) { 8 | outerQuery.filter(fq -> fq 9 | .geoDistance(gd -> gd 10 | .field("coordinate") 11 | .location(l -> l.latlon(ll -> ll.lat(location.getY()).lon(location.getX()))) 12 | .distance(radius + "km"))); 13 | } 14 | 15 | public void addQueryFilter(String query) { 16 | if (query != null && !query.isEmpty()) { 17 | outerQuery.must(qst -> qst.queryString(qs -> qs 18 | .query(query) 19 | )); 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /website/photon/static/js/app.js: -------------------------------------------------------------------------------- 1 | var searchPoints = L.geoJson(null, { 2 | onEachFeature: function (feature, layer) { 3 | layer.bindPopup(feature.properties.name); 4 | } 5 | }); 6 | function showSearchPoints (geojson) { 7 | searchPoints.clearLayers(); 8 | searchPoints.addData(geojson); 9 | } 10 | var map = L.map('map', {scrollWheelZoom: false, zoomControl: false, photonControl: true, photonControlOptions: {resultsHandler: showSearchPoints, placeholder: 'Try me…', position: 'topleft', url: API_URL}}); 11 | map.setView(CENTER, 12); 12 | searchPoints.addTo(map); 13 | var tilelayer = L.tileLayer(TILELAYER, {maxZoom: MAXZOOM, attribution: 'Data \u00a9 OpenStreetMap Contributors Tiles \u00a9 Komoot'}).addTo(map); 14 | var zoomControl = new L.Control.Zoom({position: 'topright'}).addTo(map); 15 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/searcher/PhotonResult.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.searcher; 2 | 3 | import java.util.Map; 4 | 5 | /** 6 | * Interface describing a single response object from the database. 7 | */ 8 | public interface PhotonResult { 9 | final double[] INVALID_COORDINATES = new double[]{0, 0}; 10 | 11 | /** 12 | * Get the value for the given field. 13 | * 14 | * Should throw an exception when the field has multiple values. 15 | * 16 | * @param key 17 | * @return If the field exist, the string value of the field, else null. 18 | */ 19 | Object get(String key); 20 | 21 | String getLocalised(String key, String language); 22 | Map getMap(String key); 23 | 24 | double[] getCoordinates(); 25 | 26 | String getGeometry(); 27 | 28 | double[] getExtent(); 29 | 30 | double getScore(); 31 | 32 | Map getRawData(); 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/nominatim/model/ContextMap.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.nominatim.model; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | import java.util.*; 6 | 7 | public class ContextMap extends AbstractMap> { 8 | private final Map> entries = new HashMap<>(); 9 | 10 | public void addName(String key, String name) { 11 | if (name != null) { 12 | entries.computeIfAbsent(key, k -> new HashSet<>()).add(name); 13 | } 14 | } 15 | 16 | public void addAll(Map map) { 17 | for (var entry: map.entrySet()) { 18 | addName(entry.getKey(), entry.getValue()); 19 | } 20 | } 21 | 22 | public void addAll(ContextMap map) { 23 | for (var entry : map.entrySet()) { 24 | entries.computeIfAbsent(entry.getKey(), k -> new HashSet<>()).addAll(entry.getValue()); 25 | } 26 | } 27 | 28 | @NotNull 29 | @Override 30 | public Set>> entrySet() { 31 | return entries.entrySet(); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/test/java/de/komoot/photon/ServerDatabasePropertiesTest.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon; 2 | 3 | import org.junit.jupiter.api.Test; 4 | import org.junit.jupiter.api.io.TempDir; 5 | 6 | import java.io.IOException; 7 | import java.nio.file.Path; 8 | import java.util.Date; 9 | 10 | import static org.junit.jupiter.api.Assertions.*; 11 | 12 | class ServerDatabasePropertiesTest extends ESBaseTester { 13 | 14 | @Test 15 | void testSaveAndLoadFromDatabase(@TempDir Path dataDirectory) throws IOException { 16 | setUpES(dataDirectory); 17 | 18 | final Date now = new Date(); 19 | 20 | DatabaseProperties prop = new DatabaseProperties(); 21 | prop.setLanguages(new String[]{"en", "de", "fr"}); 22 | prop.setImportDate(now); 23 | prop.setSupportGeometries(true); 24 | 25 | getServer().saveToDatabase(prop); 26 | 27 | prop = getServer().loadFromDatabase(); 28 | 29 | assertArrayEquals(new String[]{"en", "de", "fr"}, prop.getLanguages()); 30 | assertEquals(now, prop.getImportDate()); 31 | assertTrue(prop.getSupportGeometries()); 32 | } 33 | } -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/ConfigSynonyms.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon; 2 | 3 | import com.fasterxml.jackson.annotation.JsonProperty; 4 | 5 | import java.util.List; 6 | import java.util.stream.Collectors; 7 | 8 | public class ConfigSynonyms { 9 | 10 | private List searchSynonyms = null; 11 | private List classificationTerms = List.of(); 12 | 13 | public List getSearchSynonyms() { 14 | return searchSynonyms; 15 | } 16 | 17 | @JsonProperty("search_synonyms") 18 | public void setSearchSynonyms(List searchSynonyms) { 19 | this.searchSynonyms = searchSynonyms; 20 | } 21 | 22 | public List getClassificationTerms() { 23 | return classificationTerms; 24 | } 25 | 26 | @JsonProperty("classification_terms") 27 | public void setClassificationTerms(List classificationTerms) { 28 | this.classificationTerms = classificationTerms.stream() 29 | .filter(ConfigClassificationTerm::isValidCategory) 30 | .collect(Collectors.toList()); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/ConfigClassificationTerm.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon; 2 | 3 | import java.util.regex.Pattern; 4 | 5 | public class ConfigClassificationTerm { 6 | private static final Pattern LABEL_PATTERN = Pattern.compile( 7 | String.format("[%s]+", PhotonDoc.CATEGORY_VALID_CHARS) 8 | ); 9 | 10 | private String key; 11 | private String value; 12 | private String[] terms; 13 | 14 | public String getKey() { 15 | return key; 16 | } 17 | 18 | public void setKey(String key) { 19 | this.key = key; 20 | } 21 | 22 | public String getValue() { 23 | return value; 24 | } 25 | 26 | public void setValue(String value) { 27 | this.value = value; 28 | } 29 | 30 | public String[] getTerms() { 31 | return terms; 32 | } 33 | 34 | public void setTerms(String[] terms) { 35 | this.terms = terms; 36 | } 37 | 38 | public boolean isValidCategory() { 39 | return LABEL_PATTERN.matcher(key).matches() && LABEL_PATTERN.matcher(value).matches(); 40 | } 41 | 42 | public String getClassificationString() { 43 | return String.format("#osm.%s.%s", key, value); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/query/SimpleSearchRequestFactory.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.query; 2 | 3 | import io.javalin.http.Context; 4 | 5 | import java.util.List; 6 | import java.util.Set; 7 | import java.util.stream.Collectors; 8 | import java.util.stream.Stream; 9 | 10 | public class SimpleSearchRequestFactory extends SearchRequestFactoryBase implements RequestFactory { 11 | private static final Set FREE_SEARCH_PARAMETERS = 12 | Stream.concat(SEARCH_PARAMETERS.stream(), Stream.of("q")) 13 | .collect(Collectors.toSet()); 14 | 15 | public SimpleSearchRequestFactory(List supportedLanguages, String defaultLanguage, int maxResults, boolean supportGeometries) { 16 | super(supportedLanguages, defaultLanguage, maxResults, supportGeometries); 17 | } 18 | 19 | public SimpleSearchRequest create(Context context) { 20 | checkParams(context, FREE_SEARCH_PARAMETERS); 21 | 22 | final var request = new SimpleSearchRequest(); 23 | 24 | completeSearchRequest(request, context); 25 | request.setQuery(context.queryParamAsClass("q", String.class).get()); 26 | 27 | return request; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/nominatim/DBDataAdapter.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.nominatim; 2 | 3 | import org.locationtech.jts.geom.Geometry; 4 | import org.springframework.jdbc.core.JdbcTemplate; 5 | 6 | import java.sql.ResultSet; 7 | import java.sql.SQLException; 8 | import java.util.Map; 9 | 10 | /** 11 | * Defines utility functions to parse data from the database and create SQL queries. 12 | */ 13 | public interface DBDataAdapter { 14 | /** 15 | * Create a hash map from the given column data. 16 | */ 17 | Map getMap(ResultSet rs, String columnName) throws SQLException; 18 | 19 | /** 20 | * Create a JTS geometry from the given column data. 21 | */ 22 | Geometry extractGeometry(ResultSet rs, String columnName) throws SQLException; 23 | 24 | /** 25 | * Check if a table has the given column. 26 | */ 27 | boolean hasColumn(JdbcTemplate template, String table, String column); 28 | 29 | /** 30 | * Wrap a DELETE statement with a RETURNING clause. 31 | */ 32 | String deleteReturning(String deleteSQL, String columns); 33 | 34 | /** 35 | * Wrap function to create a json array from a SELECT. 36 | */ 37 | String jsonArrayFromSelect(String valueSQL, String fromSQL); 38 | } 39 | -------------------------------------------------------------------------------- /src/test/java/de/komoot/photon/nominatim/testdb/CollectingUpdater.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.nominatim.testdb; 2 | 3 | import de.komoot.photon.PhotonDoc; 4 | import de.komoot.photon.Updater; 5 | import org.assertj.core.api.ListAssert; 6 | 7 | import java.util.ArrayList; 8 | import java.util.List; 9 | 10 | import static org.assertj.core.api.Assertions.*; 11 | 12 | public class CollectingUpdater implements Updater { 13 | private final List created = new ArrayList<>(); 14 | private final List deleted = new ArrayList<>(); 15 | private int finishCalled = 0; 16 | 17 | @Override 18 | public void addOrUpdate(Iterable docs) 19 | { 20 | for (var doc: docs) { 21 | created.add(doc); 22 | } 23 | } 24 | 25 | @Override 26 | public void delete(long docId) { 27 | deleted.add(docId); 28 | } 29 | 30 | @Override 31 | public void finish() { ++finishCalled; } 32 | 33 | public int getFinishCalled() { 34 | return finishCalled; 35 | } 36 | 37 | public ListAssert assertThatDeleted() { 38 | return assertThat(deleted); 39 | } 40 | 41 | public ListAssert assertThatCreated() { 42 | return assertThat(created); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /continuously_update_from_nominatim.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | # For Nominatim < 3.7 set this to the Nominatim build directory. 4 | # For newer versions, this must be the project directory of your import. 5 | : ${NOMINATIM_DIR:=.} 6 | 7 | while true 8 | do 9 | starttime=`date +%s` 10 | 11 | # First consume updates in the Nominatim database. 12 | # The important part here is to leave out the indexing step. This 13 | # will be handled by Photon. 14 | 15 | nominatim replication --project-dir $NOMINATIM_DIR --once 16 | 17 | # Now tell Photon to finish the updates and copy the new data into its 18 | # own database. 19 | curl http://localhost:2322/nominatim-update 20 | 21 | # Sleep a bit if updates take less than a minute. 22 | # If you consume hourly or daily diffs adapt the period accordingly. 23 | endtime=`date +%s` 24 | elapsed=$((endtime - starttime)) 25 | if [[ $elapsed -lt 60 ]] 26 | then 27 | sleepy=$((60 - $elapsed)) 28 | echo "Sleeping for ${sleepy}s..." 29 | sleep $sleepy 30 | fi 31 | 32 | # Now check if the updates have finished 33 | while [ `curl -s http://localhost:2322/nominatim-update/status` != '"OK"' ]; 34 | do 35 | echo "Sleeping 15 more seconds." 36 | sleep 15 37 | done 38 | done 39 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/Constants.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon; 2 | 3 | /** 4 | * Various string constants. 5 | */ 6 | public class Constants { 7 | public static final String POSTCODE = "postcode"; 8 | public static final String HOUSENUMBER = "housenumber"; 9 | public static final String NAME = "name"; 10 | public static final String COUNTRY = "country"; 11 | public static final String COUNTRYCODE = "countrycode"; 12 | public static final String CITY = "city"; 13 | public static final String DISTRICT = "district"; 14 | public static final String LOCALITY = "locality"; 15 | public static final String STREET = "street"; 16 | public static final String STATE = "state"; 17 | public static final String COUNTY = "county"; 18 | public static final String LAT = "lat"; 19 | public static final String LON = "lon"; 20 | public static final String IMPORTANCE = "importance"; 21 | public static final String OSM_ID = "osm_id"; 22 | public static final String OSM_TYPE = "osm_type"; 23 | public static final String OSM_KEY = "osm_key"; 24 | public static final String OSM_VALUE = "osm_value"; 25 | public static final String OBJECT_TYPE = "type"; 26 | public static final String CLASSIFICATION = "classification"; 27 | public static final String GEOMETRY = "geometry"; 28 | } 29 | -------------------------------------------------------------------------------- /src/test/java/de/komoot/photon/ReflectionTestUtil.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon; 2 | 3 | import java.lang.reflect.Field; 4 | 5 | /** 6 | * Created by Sachin Dole on 2/12/2015. 7 | */ 8 | public class ReflectionTestUtil { 9 | 10 | public static T getFieldValue(Object anObject, String fieldName) { 11 | return ReflectionTestUtil.getFieldValue(anObject, anObject.getClass(), fieldName); 12 | } 13 | 14 | public static T getFieldValue(Object anObject, Class clazz, String fieldName) { 15 | try { 16 | Field path = clazz.getDeclaredField(fieldName); 17 | path.setAccessible(true); 18 | return (T) path.get(anObject); 19 | } catch (IllegalAccessException | NoSuchFieldException e) { 20 | throw new RuntimeException("unable to get value of field", e); 21 | } 22 | } 23 | 24 | public static void setFieldValue(Object anObject, Class clazz, String fieldName, T value) { 25 | try { 26 | Field path = clazz.getDeclaredField(fieldName); 27 | path.setAccessible(true); 28 | path.set(anObject, value); 29 | } catch (IllegalAccessException e) { 30 | throw new RuntimeException("unable to get value of field (illegal)", e); 31 | } catch (NoSuchFieldException e) { 32 | throw new RuntimeException("unable to get value of field (missing)", e); 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/opensearch/CategoryFilter.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.opensearch; 2 | 3 | import org.opensearch.client.opensearch._types.FieldValue; 4 | import org.opensearch.client.opensearch._types.query_dsl.Query; 5 | 6 | import java.util.Arrays; 7 | import java.util.List; 8 | import java.util.stream.Collectors; 9 | 10 | public class CategoryFilter { 11 | private final List categories; 12 | 13 | public CategoryFilter(String filterTerm) { 14 | this.categories = Arrays.stream(filterTerm.split(",")) 15 | .map(FieldValue::of) 16 | .collect(Collectors.toList()); 17 | } 18 | 19 | public Query buildIncludeQuery() { 20 | return Query.of(fn -> fn.terms(t -> t 21 | .field("categories") 22 | .terms(tm -> tm.value(categories) 23 | )) 24 | ); 25 | } 26 | 27 | public Query buildExcludeQuery() { 28 | return Query.of(fn -> fn.bool(outer -> outer 29 | .should(categories.stream() 30 | .map(s -> Query.of(q -> q.bool(inner -> inner 31 | .mustNot(m -> m.term(t -> t 32 | .field("categories") 33 | .value(s) 34 | )) 35 | ))) 36 | .collect(Collectors.toList()))) 37 | ); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/opensearch/NameCollector.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.opensearch; 2 | 3 | import java.util.Collection; 4 | import java.util.HashMap; 5 | import java.util.Map; 6 | import java.util.stream.Collectors; 7 | 8 | /** 9 | * Collects strings with a priorities, discarding duplicate strings while 10 | * keeping the highest priority. 11 | */ 12 | public class NameCollector { 13 | 14 | private final Map terms = new HashMap<>(); 15 | 16 | public NameCollector() { 17 | } 18 | 19 | public NameCollector(Collection termCollection) { 20 | addAll(termCollection, 1); 21 | } 22 | 23 | public void add(String term, int searchPrio) { 24 | final var cleaned = term.replace("|", " "); 25 | final var currentPrio = terms.get(cleaned); 26 | if (currentPrio == null || currentPrio < searchPrio) { 27 | terms.put(cleaned, Integer.max(searchPrio, 1)); 28 | } 29 | } 30 | 31 | public void addAll(Collection termCollection, int searchPrio) { 32 | for (var term : termCollection) { 33 | add(term, searchPrio); 34 | } 35 | } 36 | 37 | public String toCollectorString() { 38 | return terms.entrySet().stream() 39 | .sorted((e1, e2) -> e2.getValue().compareTo(e1.getValue())) 40 | .map(e -> String.format("%s|%d", e.getKey(), e.getValue())) 41 | .collect(Collectors.joining(";")); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/query/ReverseRequest.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.query; 2 | 3 | import org.locationtech.jts.geom.Point; 4 | 5 | /** 6 | * Collection of query parameters for a reverse request. 7 | */ 8 | public class ReverseRequest extends RequestBase { 9 | private Point location; 10 | private double radius = 1.0; 11 | private String queryStringFilter; 12 | private boolean locationDistanceSort = true; 13 | 14 | public ReverseRequest() { 15 | setLimit(1, 1); 16 | } 17 | 18 | public Point getLocation() { 19 | return location; 20 | } 21 | 22 | public double getRadius() { 23 | return radius; 24 | } 25 | 26 | public String getQueryStringFilter() { 27 | return queryStringFilter; 28 | } 29 | 30 | public boolean getLocationDistanceSort() { 31 | return locationDistanceSort; 32 | } 33 | 34 | public void setLocation(Point location) { 35 | this.location = location; 36 | } 37 | 38 | public void setRadius(Double radius) { 39 | if (radius != null) { 40 | this.radius = radius; 41 | } 42 | } 43 | 44 | public void setQueryStringFilter(String queryStringFilter) { 45 | this.queryStringFilter = queryStringFilter; 46 | } 47 | 48 | public void setLocationDistanceSort(Boolean locationDistanceSort) { 49 | if (locationDistanceSort != null) { 50 | this.locationDistanceSort = locationDistanceSort; 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/test/java/de/komoot/photon/PhotonDocTest.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon; 2 | 3 | import de.komoot.photon.nominatim.model.AddressType; 4 | import org.junit.jupiter.api.Test; 5 | 6 | import java.util.Map; 7 | 8 | import static org.assertj.core.api.Assertions.*; 9 | 10 | class PhotonDocTest { 11 | 12 | private PhotonDoc simplePhotonDoc() { 13 | return new PhotonDoc(1, "W", 2, "highway", "residential").houseNumber("4"); 14 | } 15 | 16 | @Test 17 | void testCompleteAddressOverwritesStreet() { 18 | PhotonDoc doc = simplePhotonDoc(); 19 | 20 | doc.setAddressPartIfNew(AddressType.STREET, Map.of("name", "parent place street")); 21 | doc.addAddresses(Map.of("street", "test street"), new String[]{"de"}); 22 | 23 | assertThat(doc.getAddressParts().get(AddressType.STREET)) 24 | .containsEntry("default", "test street"); 25 | } 26 | 27 | @Test 28 | void testCompleteAddressCreatesStreetIfNonExistantBefore() { 29 | PhotonDoc doc = simplePhotonDoc(); 30 | 31 | doc.addAddresses(Map.of("street", "test street"), new String[]{"de"}); 32 | 33 | assertThat(doc.getAddressParts().get(AddressType.STREET)) 34 | .containsEntry("default", "test street"); 35 | 36 | } 37 | 38 | @Test 39 | void testAddCountryCode() { 40 | PhotonDoc doc = new PhotonDoc(1, "W", 2, "highway", "residential").countryCode("de"); 41 | 42 | assertThat(doc.getCountryCode()) 43 | .isEqualTo("DE"); 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/query/ReverseRequestFactory.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.query; 2 | 3 | import io.javalin.http.Context; 4 | 5 | import java.util.List; 6 | import java.util.Set; 7 | import java.util.stream.Collectors; 8 | import java.util.stream.Stream; 9 | 10 | public class ReverseRequestFactory extends RequestFactoryBase implements RequestFactory { 11 | private static final Set REVERSE_PARAMETERS = 12 | Stream.concat(BASE_PARAMETERS.stream(), 13 | Stream.of("lat", "lon", "radius", "query_string_filter", "distance_sort")) 14 | .collect(Collectors.toSet()); 15 | 16 | public ReverseRequestFactory(List supportedLanguages, String defaultLanguage, int maxResults, boolean supportGeometries) { 17 | super(supportedLanguages, defaultLanguage, maxResults, supportGeometries); 18 | } 19 | 20 | public ReverseRequest create(Context context) { 21 | checkParams(context, REVERSE_PARAMETERS); 22 | 23 | final var request = new ReverseRequest(); 24 | 25 | completeBaseRequest(request, context); 26 | request.setLocation(parseLatLon(context, true)); 27 | request.setRadius(context.queryParamAsClass("radius", Double.class).allowNullable().get()); 28 | request.setQueryStringFilter(context.queryParam("query_string_filter")); 29 | request.setLocationDistanceSort(context.queryParamAsClass("distance_sort", Boolean.class).getOrDefault(true)); 30 | 31 | return request; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/test/java/de/komoot/photon/metrics/MetricsConfigTest.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.metrics; 2 | 3 | import de.komoot.photon.CommandLineArgs; 4 | import org.junit.jupiter.api.Test; 5 | import org.opensearch.client.opensearch.OpenSearchClient; 6 | 7 | import static org.junit.jupiter.api.Assertions.*; 8 | 9 | class MetricsConfigTest { 10 | @Test 11 | void testInit() { 12 | CommandLineArgs args = new CommandLineArgs() { 13 | @Override 14 | public String getMetricsEnable() { 15 | return "prometheus"; 16 | } 17 | }; 18 | OpenSearchClient openSearchClient = new OpenSearchClient(null) {}; 19 | 20 | MetricsConfig metricsConfig = MetricsConfig.setupMetrics(args, openSearchClient); 21 | assertNotNull(metricsConfig.getRegistry()); 22 | assertNotNull(metricsConfig.getPlugin()); 23 | assertTrue(metricsConfig.isEnabled()); 24 | } 25 | 26 | @Test 27 | void testNoInit() { 28 | CommandLineArgs args = new CommandLineArgs() { 29 | @Override 30 | public String getMetricsEnable() { 31 | return ""; 32 | } 33 | }; 34 | OpenSearchClient openSearchClient = new OpenSearchClient(null) {}; 35 | 36 | MetricsConfig metricsConfig = MetricsConfig.setupMetrics(args, openSearchClient); 37 | assertThrows(IllegalStateException.class, metricsConfig::getRegistry); 38 | assertThrows(IllegalStateException.class, metricsConfig::getPlugin); 39 | assertFalse(metricsConfig.isEnabled()); 40 | } 41 | 42 | } -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/PhotonDocInterpolationSet.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | import org.locationtech.jts.geom.Geometry; 5 | import org.locationtech.jts.linearref.LengthIndexedLine; 6 | 7 | import java.util.ArrayList; 8 | import java.util.Iterator; 9 | import java.util.List; 10 | 11 | public class PhotonDocInterpolationSet implements Iterable { 12 | 13 | private final List docs = new ArrayList<>(); 14 | 15 | public PhotonDocInterpolationSet(PhotonDoc base, long first, long last, long step, Geometry geom) { 16 | base.bbox(geom.getEnvelope()); 17 | if (last == first) { 18 | docs.add(base.houseNumber(String.valueOf(first)).centroid(geom.getCentroid())); 19 | } else if (first < last && (last - first) < 600 && step < 10) { 20 | LengthIndexedLine line = new LengthIndexedLine(geom); 21 | double si = line.getStartIndex(); 22 | double ei = line.getEndIndex(); 23 | double lstep = (ei - si) / (last - first); 24 | 25 | var fac = geom.getFactory(); 26 | for (long num = 0; first + num <= last; num += step) { 27 | docs.add(new PhotonDoc(base) 28 | .houseNumber(String.valueOf(num + first)) 29 | .centroid(fac.createPoint(line.extractPoint(si + lstep * num))) 30 | ); 31 | } 32 | } 33 | } 34 | 35 | @Override 36 | public @NotNull Iterator iterator() { 37 | return docs.iterator(); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/test/java/de/komoot/photon/nominatim/model/NameMapTest.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.nominatim.model; 2 | 3 | import org.junit.jupiter.params.ParameterizedTest; 4 | import org.junit.jupiter.params.provider.Arguments; 5 | import org.junit.jupiter.params.provider.MethodSource; 6 | 7 | import java.util.Map; 8 | import java.util.stream.Stream; 9 | 10 | import static org.assertj.core.api.Assertions.*; 11 | import static org.junit.jupiter.params.provider.Arguments.arguments; 12 | 13 | public class NameMapTest { 14 | 15 | @ParameterizedTest 16 | @MethodSource("validPlaceNameProvider") 17 | void testPlaceName(Map input, Map output) { 18 | assertThat(NameMap.makeForPlace(input, new String[]{"en", "it"})) 19 | .isEqualTo(output); 20 | } 21 | 22 | static Stream validPlaceNameProvider() { 23 | return Stream.of( 24 | arguments( 25 | Map.of("name", "ABC"), 26 | Map.of("default", "ABC")), 27 | arguments( 28 | Map.of("name", "ABC", "_place_name", "CBA"), 29 | Map.of("default", "CBA")), 30 | arguments( 31 | Map.of("name", "standard", "name:en", "foo", "name:de:it", "fooi", "name:ch", "f"), 32 | Map.of("default", "standard", "en", "foo")), 33 | arguments( 34 | Map.of("name", "this", "alt_name", "that", "old_name", "dis"), 35 | Map.of("default", "this", "alt", "that", "old", "dis")) 36 | ); 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /src/test/java/de/komoot/photon/searcher/StreetDupesRemoverTest.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.searcher; 2 | 3 | import de.komoot.photon.Constants; 4 | import org.junit.jupiter.api.Test; 5 | 6 | import java.util.List; 7 | 8 | import static org.assertj.core.api.Assertions.*; 9 | 10 | class StreetDupesRemoverTest { 11 | 12 | @Test 13 | void testDeduplicatesStreets() { 14 | StreetDupesRemover streetDupesRemover = new StreetDupesRemover("en"); 15 | var allResults = List.of( 16 | createDummyResult("99999", "Main Street", "highway", "Unclassified"), 17 | createDummyResult("99999", "Main Street", "highway", "Unclassified")); 18 | 19 | assertThat(streetDupesRemover.execute(allResults)) 20 | .hasSize(1); 21 | } 22 | 23 | @Test 24 | void testStreetAndBusStopNotDeduplicated() { 25 | StreetDupesRemover streetDupesRemover = new StreetDupesRemover("en"); 26 | var allResults = List.of( 27 | createDummyResult("99999", "Main Street", "highway", "bus_stop"), 28 | createDummyResult("99999", "Main Street", "highway", "Unclassified")); 29 | 30 | assertThat(streetDupesRemover.execute(allResults)) 31 | .hasSize(2); 32 | } 33 | 34 | private PhotonResult createDummyResult(String postCode, String name, String osmKey, 35 | String osmValue) { 36 | return new MockPhotonResult() 37 | .put(Constants.POSTCODE, postCode) 38 | .putLocalized(Constants.NAME, "en", name) 39 | .put(Constants.OSM_KEY, osmKey) 40 | .put(Constants.OSM_VALUE, osmValue); 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/query/SearchRequestBase.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.query; 2 | 3 | import org.locationtech.jts.geom.Envelope; 4 | import org.locationtech.jts.geom.Point; 5 | 6 | public class SearchRequestBase extends RequestBase { 7 | private Point locationForBias = null; 8 | private double scale = 0.2; 9 | private int zoom = 14; 10 | private Envelope bbox = null; 11 | private boolean suggestAddresses = false; 12 | 13 | public Envelope getBbox() { 14 | return bbox; 15 | } 16 | 17 | public Point getLocationForBias() { 18 | return locationForBias; 19 | } 20 | 21 | public double getScaleForBias() { 22 | return scale; 23 | } 24 | 25 | public int getZoomForBias() { 26 | return zoom; 27 | } 28 | 29 | void setLocationForBias(Point locationForBias) { 30 | if (locationForBias != null) { 31 | this.locationForBias = locationForBias; 32 | } 33 | } 34 | 35 | void setScale(Double scale) { 36 | if (scale != null) { 37 | this.scale = Double.max(Double.min(scale, 1.0), 0.0); 38 | } 39 | } 40 | 41 | void setZoom(Integer zoom) { 42 | if (zoom != null) { 43 | this.zoom = Integer.max(Integer.min(zoom, 18), 0); 44 | } 45 | } 46 | 47 | void setBbox(Envelope bbox) { 48 | if (bbox != null) { 49 | this.bbox = bbox; 50 | } 51 | } 52 | 53 | public boolean getSuggestAddresses() { 54 | return suggestAddresses; 55 | } 56 | 57 | public void setSuggestAddresses(boolean suggestAddresses) { 58 | this.suggestAddresses = suggestAddresses; 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/query/SearchRequestFactoryBase.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.query; 2 | 3 | import io.javalin.http.Context; 4 | import org.locationtech.jts.geom.Envelope; 5 | 6 | import java.util.List; 7 | import java.util.Set; 8 | import java.util.stream.Collectors; 9 | import java.util.stream.Stream; 10 | 11 | public class SearchRequestFactoryBase extends RequestFactoryBase { 12 | protected static final Set SEARCH_PARAMETERS = 13 | Stream.concat(BASE_PARAMETERS.stream(), 14 | Stream.of("lat", "lon", "location_bias_scale", "zoom", "bbox", "suggest_addresses")) 15 | .collect(Collectors.toSet()); 16 | 17 | protected SearchRequestFactoryBase(List supportedLanguages, String defaultLanguage, int maxResults, boolean supportGeometries) { 18 | super(supportedLanguages, defaultLanguage, maxResults, supportGeometries); 19 | } 20 | 21 | protected void completeSearchRequest(SearchRequestBase request, Context context) { 22 | completeBaseRequest(request, context); 23 | 24 | request.setZoom(context.queryParamAsClass("zoom", Integer.class) 25 | .allowNullable().get()); 26 | 27 | request.setScale(context.queryParamAsClass("location_bias_scale", Double.class) 28 | .allowNullable() 29 | .get()); 30 | 31 | request.setLocationForBias(parseLatLon(context, false)); 32 | request.setBbox(context.queryParamAsClass("bbox", Envelope.class) 33 | .allowNullable().get()); 34 | request.setSuggestAddresses(context.queryParamAsClass("suggest_addresses", Boolean.class) 35 | .getOrDefault(false)); 36 | } 37 | 38 | } 39 | -------------------------------------------------------------------------------- /src/test/java/de/komoot/photon/DatabasePropertiesTest.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon; 2 | 3 | import org.junit.jupiter.api.Test; 4 | 5 | import static org.junit.jupiter.api.Assertions.*; 6 | 7 | import java.util.Date; 8 | 9 | /** 10 | * Tests for the database-global property store. 11 | */ 12 | class DatabasePropertiesTest { 13 | 14 | /** 15 | * setLanguages() overwrites the language settings. 16 | */ 17 | @Test 18 | void testSetLanguages() { 19 | var now = new Date(); 20 | DatabaseProperties prop = new DatabaseProperties(); 21 | prop.setLanguages(new String[]{"en", "bg", "de"}); 22 | prop.setImportDate(now); 23 | 24 | assertArrayEquals(new String[]{"en", "bg", "de"}, prop.getLanguages()); 25 | assertEquals(now, prop.getImportDate()); 26 | } 27 | 28 | /** 29 | * If languages is not set, then the restricted language set is used as is. 30 | */ 31 | @Test 32 | void testRestrictLanguagesUnsetLanguages() { 33 | DatabaseProperties prop = new DatabaseProperties(); 34 | prop.restrictLanguages(new String[]{"en", "bg", "de"}); 35 | 36 | assertArrayEquals(new String[]{"en", "bg", "de"}, prop.getLanguages()); 37 | } 38 | 39 | /** 40 | * When languages are set, then only the languages of the restricted set are used 41 | * that already exist and the order of the input is preserved. 42 | */ 43 | @Test 44 | void testRestrictLanguagesAlreadySet() { 45 | DatabaseProperties prop = new DatabaseProperties(); 46 | prop.setLanguages(new String[]{"en", "de", "fr"}); 47 | 48 | prop.restrictLanguages(new String[]{"cn", "de", "en", "es"}); 49 | 50 | assertArrayEquals(new String[]{"de", "en"}, prop.getLanguages()); 51 | } 52 | 53 | } 54 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/json/DumpFields.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.json; 2 | 3 | public class DumpFields { 4 | 5 | private DumpFields() {} 6 | 7 | public static final String HEADER_VERSION = "version"; 8 | public static final String HEADER_GENERATOR = "generator"; 9 | public static final String HEADER_DB_VERSION = "database_version"; 10 | public static final String HEADER_DB_TIME = "data_timestamp"; 11 | public static final String HEADER_FEATURES = "features"; 12 | 13 | public static final String DOCUMENT_TYPE = "type"; 14 | public static final String DOCUMENT_CONTENT = "content"; 15 | 16 | public static final String PLACE_ID = "place_id"; 17 | public static final String PLACE_OBJECT_TYPE = "object_type"; 18 | public static final String PLACE_OBJECT_ID = "object_id"; 19 | public static final String PLACE_OSM_KEY = "osm_key"; 20 | public static final String PLACE_OSM_VALUE = "osm_value"; 21 | public static final String PLACE_CATEGORIES = "categories"; 22 | public static final String PLACE_RANK_ADDRESS = "rank_address"; 23 | public static final String PLACE_IMPORTANCE = "importance"; 24 | public static final String PLACE_NAMES = "name"; 25 | public static final String PLACE_HOUSENUMBER = "housenumber"; 26 | public static final String PLACE_ADDRESS = "address"; 27 | public static final String PLACE_EXTRA_TAGS = "extra"; 28 | public static final String PLACE_POSTCODE = "postcode"; 29 | public static final String PLACE_COUNTRY_CODE = "country_code"; 30 | public static final String PLACE_CENTROID = "centroid"; 31 | public static final String PLACE_BBOX = "bbox"; 32 | public static final String PLACE_GEOMETRY = "geometry"; 33 | public static final String PLACE_ADDRESSLINES = "addresslines"; 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/query/BoundingBoxParamConverter.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.query; 2 | 3 | import org.locationtech.jts.geom.Envelope; 4 | 5 | /** 6 | * Converter which transforms a bbox parameter into an Envelope and performs format checking. 7 | */ 8 | public class BoundingBoxParamConverter { 9 | 10 | public static final String INVALID_BBOX_ERROR_MESSAGE = "Invalid number of supplied coordinates for parameter 'bbox', expected format is: minLon,minLat,maxLon,maxLat"; 11 | public static final String INVALID_BBOX_BOUNDS_MESSAGE = "Invalid bounds for parameter 'bbox', expected values minLat, maxLat element [-90,90], minLon, maxLon element [-180,180]"; 12 | 13 | public static Envelope apply(String bboxParam) { 14 | if (bboxParam == null) { 15 | return null; 16 | } 17 | 18 | String[] bboxCoords = bboxParam.split(","); 19 | if (bboxCoords.length != 4) { 20 | throw new BadRequestException(400, INVALID_BBOX_ERROR_MESSAGE); 21 | } 22 | 23 | return new Envelope(parseDouble(bboxCoords[0], 180), 24 | parseDouble(bboxCoords[2], 180), 25 | parseDouble(bboxCoords[1], 90), 26 | parseDouble(bboxCoords[3], 90)); 27 | } 28 | 29 | private static double parseDouble(String coord, double limit) { 30 | double result; 31 | try { 32 | result = Double.parseDouble(coord); 33 | } catch (NumberFormatException nfe) { 34 | throw new BadRequestException(400, INVALID_BBOX_ERROR_MESSAGE); 35 | } 36 | 37 | if (Double.isNaN(result) || result < -limit || result > limit) { 38 | throw new BadRequestException(400, INVALID_BBOX_BOUNDS_MESSAGE); 39 | } 40 | 41 | return result; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /docs/structured.md: -------------------------------------------------------------------------------- 1 | # Using structured queries 2 | 3 | The OpenSearch version of Photon has a separate endpoint for structured queries. 4 | Structured queries make it possible to search for specific 5 | countrycode / city / postcode / street / ... instead of the default query string. 6 | If the address fields are known, structured queries often lead to better 7 | results than the construction of a query string followed by a free text search. 8 | 9 | ## Enabling support 10 | 11 | **Starting from Photon 1.0, structured queries are always available.** 12 | 13 | Use "-structured" when importing the nominatim database. This option increases the index size by around 10%. 14 | 15 | The information whether the data was imported with structured query support is stored within the OpenSearch index. 16 | 17 | On startup photon checks whether the index supports structured queries and if so 18 | ``` 19 | http://localhost:2322/structured?city=berlin 20 | ``` 21 | is available. 22 | 23 | ## Usage 24 | 25 | Supported parameters are 26 | ``` 27 | "lang", "limit", "lon", "lat", "osm_tag", "location_bias_scale", "bbox", "debug", "zoom", "layer", "countrycode", "state", "county", "city", "postcode", "district", "housenumber", "street" 28 | ``` 29 | 30 | countrycode has to be a valid ISO 3166-1 alpha-2 code (also known as ISO2). 31 | All parameters shared with /api have the same meaning. 32 | 33 | The result format is the same as for /api. 34 | 35 | ## Known issues 36 | 37 | * state information is used with low priority. This can cause issues with cities that exist in several states (e.g. "Springfield" in the US). The reason is that states are not normalized - some documents have abbreviations like "NY", other spell "New York" out. 38 | * abbreviations like the English 'Ave' (Avenue) and the German "Str." (Straße) are not supported. 39 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/nominatim/model/NameMap.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.nominatim.model; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | import java.util.*; 6 | 7 | public class NameMap extends AbstractMap { 8 | private final Set> entries = new HashSet<>(); 9 | 10 | @NotNull 11 | @Override 12 | public Set> entrySet() { 13 | return entries; 14 | } 15 | 16 | public static NameMap makeForPlace(Map source, String[] languages) { 17 | return new NameMap() 18 | .setLocaleNames(source, languages) 19 | .setName("alt", source, "_place_alt_name", "alt_name") 20 | .setName("int", source, "_place_int_name", "int_name") 21 | .setName("loc", source, "_place_loc_name", "loc_name") 22 | .setName("old", source, "_place_old_name", "old_name") 23 | .setName("reg", source, "_place_reg_name", "reg_name") 24 | .setName("housename", source,"addr:housename"); 25 | } 26 | 27 | NameMap setLocaleNames(Map source, String[] languages) { 28 | setName("default", source, "_place_name", "name"); 29 | for (var lang : languages) { 30 | setName(lang, source, "_place_name:" + lang, "name:" + lang); 31 | } 32 | return this; 33 | } 34 | 35 | NameMap setName(String field, Map source, String... keys) { 36 | if (!containsKey(field)) { 37 | Arrays.stream(keys) 38 | .map(source::get) 39 | .filter(Objects::nonNull) 40 | .findFirst() 41 | .ifPresent(k -> entries.add(new SimpleImmutableEntry<>(field, k))); 42 | } 43 | return this; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/opensearch/BaseQueryBuilder.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.opensearch; 2 | 3 | import de.komoot.photon.searcher.TagFilter; 4 | import org.opensearch.client.opensearch._types.FieldValue; 5 | import org.opensearch.client.opensearch._types.query_dsl.BoolQuery; 6 | import org.opensearch.client.opensearch._types.query_dsl.Query; 7 | 8 | import java.util.Collection; 9 | import java.util.List; 10 | import java.util.stream.Collectors; 11 | 12 | /** 13 | * Provides the basic query structure as well as functions to 14 | * add sub-queries for the common query parameters. 15 | */ 16 | public class BaseQueryBuilder { 17 | protected final BoolQuery.Builder outerQuery = new BoolQuery.Builder(); 18 | 19 | public Query build() { 20 | return outerQuery.build().toQuery(); 21 | } 22 | 23 | public void addOsmTagFilter(List filters) { 24 | if (!filters.isEmpty()) { 25 | outerQuery.filter(new OsmTagFilter().withOsmTagFilters(filters).build()); 26 | } 27 | } 28 | 29 | public void addLayerFilter(Collection layers) { 30 | if (!layers.isEmpty()) { 31 | outerQuery.filter(f -> f.terms(t -> t 32 | .field("type") 33 | .terms(tm -> tm.value( 34 | layers.stream().map(FieldValue::of).collect(Collectors.toList()) 35 | )) 36 | )); 37 | } 38 | } 39 | 40 | public void includeCategories(Collection queryTerms) { 41 | for (var term : queryTerms) { 42 | outerQuery.filter(new CategoryFilter(term).buildIncludeQuery()); 43 | } 44 | } 45 | 46 | public void excludeCategories(Collection queryTerms) { 47 | for (var term : queryTerms) { 48 | outerQuery.filter(new CategoryFilter(term).buildExcludeQuery()); 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/nominatim/model/OsmlineRowMapper.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.nominatim.model; 2 | 3 | import de.komoot.photon.PhotonDoc; 4 | import de.komoot.photon.nominatim.DBDataAdapter; 5 | import org.jetbrains.annotations.NotNull; 6 | import org.springframework.jdbc.core.RowMapper; 7 | 8 | import java.sql.ResultSet; 9 | import java.sql.SQLException; 10 | import java.util.List; 11 | 12 | public class OsmlineRowMapper implements RowMapper { 13 | @Override 14 | @NotNull 15 | public PhotonDoc mapRow(ResultSet rs, int rowNum) throws SQLException { 16 | return new PhotonDoc( 17 | rs.getLong("place_id"), 18 | "W", rs.getLong("osm_id"), 19 | "place", "house_number") 20 | .countryCode(rs.getString("country_code")) 21 | .categories(List.of("osm.place.house_number")) 22 | .postcode(rs.getString("postcode")); 23 | } 24 | 25 | public String makeBaseQuery(DBDataAdapter dbutils) { 26 | return "SELECT p.place_id, p.osm_id, p.startnumber, p.endnumber," + 27 | " p.postcode, p.country_code, p.address, p.linegeo, p.step," + 28 | " parent.class as parent_class, parent.type as parent_type," + 29 | " parent.rank_address as parent_rank_address, parent.name as parent_name, " + 30 | dbutils.jsonArrayFromSelect( 31 | "address_place_id", 32 | "FROM place_addressline pa " + 33 | " WHERE pa.place_id IN (p.place_id, coalesce(p.parent_place_id, p.place_id)) AND isaddress" + 34 | " ORDER BY cached_rank_address DESC, pa.place_id = p.place_id DESC") + " as addresslines" + 35 | " FROM location_property_osmline p LEFT JOIN placex parent ON p.parent_place_id = parent.place_id" + 36 | " WHERE startnumber is not null"; 37 | } 38 | } -------------------------------------------------------------------------------- /src/test/java/de/komoot/photon/searcher/TagFilterTest.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.searcher; 2 | 3 | import org.junit.jupiter.params.ParameterizedTest; 4 | import org.junit.jupiter.params.provider.Arguments; 5 | import org.junit.jupiter.params.provider.MethodSource; 6 | import org.junit.jupiter.params.provider.ValueSource; 7 | 8 | import java.util.stream.Stream; 9 | 10 | import static org.junit.jupiter.api.Assertions.*; 11 | import static org.junit.jupiter.params.provider.Arguments.arguments; 12 | 13 | class TagFilterTest { 14 | 15 | @ParameterizedTest 16 | @MethodSource("validOsmTagFilterValueProvider") 17 | void testBuildOsmTagFilterOk(String filter, TagFilterKind kind, String key, String value) { 18 | assertEquals(new TagFilter(kind, key, value), 19 | TagFilter.buildOsmTagFilter(filter)); 20 | } 21 | 22 | static Stream validOsmTagFilterValueProvider() { 23 | return Stream.of( 24 | arguments("tourism", TagFilterKind.INCLUDE, "tourism", null), 25 | arguments(":information", TagFilterKind.INCLUDE, null, "information"), 26 | arguments("shop:bakery", TagFilterKind.INCLUDE, "shop", "bakery"), 27 | arguments("!highway", TagFilterKind.EXCLUDE, "highway", null), 28 | arguments("!:path", TagFilterKind.EXCLUDE, null, "path"), 29 | arguments(":!path", TagFilterKind.EXCLUDE, null, "path"), 30 | arguments("!highway:path", TagFilterKind.EXCLUDE, "highway", "path"), 31 | arguments("!highway:!path", TagFilterKind.EXCLUDE, "highway", "path"), 32 | arguments("amenity:!post_box", TagFilterKind.EXCLUDE_VALUE, "amenity", "post_box") 33 | ); 34 | } 35 | 36 | 37 | @ParameterizedTest 38 | @ValueSource(strings = {"", ":", "addr:housenumber:1", "shop:"}) 39 | void testBuildOsmTagFilterInvalid(String filter) { 40 | assertNull(TagFilter.buildOsmTagFilter(filter)); 41 | } 42 | 43 | } -------------------------------------------------------------------------------- /src/test/java/de/komoot/photon/nominatim/testdb/H2DataAdapter.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.nominatim.testdb; 2 | 3 | import com.fasterxml.jackson.core.JsonProcessingException; 4 | import com.fasterxml.jackson.core.type.TypeReference; 5 | import com.fasterxml.jackson.databind.ObjectMapper; 6 | import org.locationtech.jts.geom.Geometry; 7 | import de.komoot.photon.nominatim.DBDataAdapter; 8 | import org.springframework.jdbc.core.JdbcTemplate; 9 | 10 | import java.sql.ResultSet; 11 | import java.sql.SQLException; 12 | import java.util.HashMap; 13 | import java.util.Map; 14 | 15 | public class H2DataAdapter implements DBDataAdapter { 16 | private static final TypeReference> mapTypeRef = new TypeReference<>() {}; 17 | private static final ObjectMapper objectMapper = new ObjectMapper(); 18 | @Override 19 | public Map getMap(ResultSet rs, String columnName) throws SQLException { 20 | String json = rs.getString(columnName); 21 | try { 22 | return json == null ? Map.of() : objectMapper.readValue(rs.getString(columnName), mapTypeRef); 23 | } catch (JsonProcessingException e) { 24 | throw new RuntimeException(e); 25 | } 26 | } 27 | 28 | @Override 29 | public Geometry extractGeometry(ResultSet rs, String columnName) throws SQLException { 30 | return (Geometry) rs.getObject(columnName); 31 | } 32 | 33 | @Override 34 | public boolean hasColumn(JdbcTemplate template, String table, String column) 35 | { 36 | if ("location_property_osmline".equals(table) && "step".equals(column)) { 37 | return true; 38 | } 39 | return false; 40 | } 41 | 42 | @Override 43 | public String deleteReturning(String deleteSQL, String columns) { 44 | return "SELECT " + columns + " FROM OLD TABLE (" + deleteSQL + ")"; 45 | } 46 | 47 | @Override 48 | public String jsonArrayFromSelect(String valueSQL, String fromSQL) { 49 | return "json_array((SELECT " + valueSQL + " " + fromSQL + ") FORMAT JSON)"; 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/test/java/de/komoot/photon/searcher/MockPhotonResult.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.searcher; 2 | 3 | import java.util.HashMap; 4 | import java.util.Map; 5 | 6 | public class MockPhotonResult implements PhotonResult { 7 | 8 | final Map data = new HashMap<>(); 9 | final double[] coordinates = new double[]{42, 21}; 10 | String geometry = "{\"type\":\"MultiPolygon\",\"coordinates\":[[[[-100.0,40.0],[-100.0,45.0],[-90.0,45.0],[-90.0,40.0],[-100.0,40.0]]],[[[-80.0,35.0],[-80.0,40.0],[-70.0,40.0],[-70.0,35.0],[-80.0,35.0]]]]}"; 11 | final double[] extent = new double[]{0, 1, 2, 3}; 12 | final Map localized = new HashMap<>(); 13 | 14 | @Override 15 | public Object get(String key) { 16 | return data.getOrDefault(key, null); 17 | } 18 | 19 | @Override 20 | public String getLocalised(String key, String language) { 21 | return localized.getOrDefault(key + "||" + language, null); 22 | } 23 | 24 | @Override 25 | public Map getMap(String key) { 26 | return (Map) data.getOrDefault(key, null); 27 | } 28 | 29 | @Override 30 | public double[] getCoordinates() { 31 | return coordinates; 32 | } 33 | 34 | @Override 35 | public String getGeometry() { 36 | return geometry; 37 | } 38 | 39 | @Override 40 | public double[] getExtent() { 41 | return extent; 42 | } 43 | 44 | @Override 45 | public double getScore() { 46 | return 99; 47 | } 48 | 49 | @Override 50 | public Map getRawData() { 51 | return Map.of(); 52 | } 53 | 54 | public MockPhotonResult put(String key, Object value) { 55 | data.put(key, value); 56 | return this; 57 | } 58 | 59 | public MockPhotonResult putLocalized(String key, String lang, String value) { 60 | localized.put(key + "||" + lang, value); 61 | return this; 62 | } 63 | 64 | public MockPhotonResult putGeometry(String geometry) { 65 | this.geometry = geometry; 66 | return this; 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /website/photon/static/js/controllers.js: -------------------------------------------------------------------------------- 1 | /*jshint globalstrict:true */ 2 | /*global angular:true */ 3 | /*global _:true */ 4 | 'use strict'; 5 | 6 | angular.module('photon.controllers', []) 7 | .controller('SearchCtrl', function($scope, $location, $http) { 8 | $scope.hits = []; 9 | $scope.searchString = ""; 10 | $scope.center = { 11 | lat: 48.8, 12 | lng: 2.7, 13 | zoom: 4 14 | }; 15 | $scope.markers = []; 16 | $scope.tiles = {url: "http://{s}.tile.openstreetmap.fr/osmfr/{z}/{x}/{y}.png"}; 17 | 18 | var getLatLng = function (hit) { 19 | var latlng = hit.coordinate.split(','); 20 | latlng = { 21 | lat: parseFloat(latlng[0], 10), 22 | lng: parseFloat(latlng[1], 10) 23 | }; 24 | return latlng; 25 | }; 26 | 27 | $scope.search = function () { 28 | $http.get('/search/?q=' + encodeURIComponent($scope.searchString), {cache: true}).success(function(data) { 29 | $scope.hits = data.docs; 30 | $scope.highlight = data.highlight; 31 | $scope.markers = _.map($scope.hits, function (hit, key, list) { 32 | return getLatLng(hit); 33 | }); 34 | }); 35 | }; 36 | 37 | $scope.mapCenter = function (hit) { 38 | var latlng = getLatLng(hit); 39 | $scope.center = { 40 | lat: latlng.lat, 41 | lng: latlng.lng, 42 | zoom: 15 43 | }; 44 | $scope.markers = [latlng]; 45 | $scope.hits = []; 46 | $scope.searchString = ""; 47 | }; 48 | 49 | $scope.getTitle = function (hit) { 50 | var title = []; 51 | var hl = $scope.highlight[hit.id]; 52 | var fields = ['name', 'street', 'city', 'country'], field; 53 | for (var i = 0; i < fields.length; i++) { 54 | field = fields[i]; 55 | if (hl[field]) { 56 | title.push(hl[field].join(' ')); 57 | } else if (hit[field]) { 58 | title.push(hit[field]); 59 | } 60 | } 61 | return title.join(', '); 62 | }; 63 | 64 | }); -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/query/StructuredSearchRequestFactory.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.query; 2 | 3 | import io.javalin.http.Context; 4 | 5 | import java.util.List; 6 | import java.util.Set; 7 | import java.util.stream.Collectors; 8 | import java.util.stream.Stream; 9 | 10 | public class StructuredSearchRequestFactory extends SearchRequestFactoryBase implements RequestFactory { 11 | private static final List STRUCTURED_ADDRESS_FIELDS = List.of( 12 | "countrycode", "state", "county", "city", 13 | "postcode", "district", "housenumber", "street"); 14 | private static final Set STRUCTURED_SEARCH_PARAMETERS = 15 | Stream.concat(SEARCH_PARAMETERS.stream(), STRUCTURED_ADDRESS_FIELDS.stream()) 16 | .collect(Collectors.toSet()); 17 | 18 | 19 | public StructuredSearchRequestFactory(List supportedLanguages, String defaultLanguage, int maxResults, boolean supportGeometries) { 20 | super(supportedLanguages, defaultLanguage, maxResults, supportGeometries); 21 | } 22 | 23 | public StructuredSearchRequest create(Context context) { 24 | checkParams(context, STRUCTURED_SEARCH_PARAMETERS); 25 | 26 | if (STRUCTURED_ADDRESS_FIELDS.stream() 27 | .noneMatch(s -> context.queryParam(s) != null)) { 28 | throw new BadRequestException(400, "at least one of the parameters " 29 | + STRUCTURED_ADDRESS_FIELDS + " is required."); 30 | } 31 | 32 | final var request = new StructuredSearchRequest(); 33 | 34 | completeSearchRequest(request, context); 35 | request.setCountryCode(context.queryParam("countrycode")); 36 | request.setState(context.queryParam("state")); 37 | request.setCounty(context.queryParam("county")); 38 | request.setCity(context.queryParam("city")); 39 | request.setPostCode(context.queryParam("postcode")); 40 | request.setDistrict(context.queryParam("district")); 41 | request.setStreet(context.queryParam("street")); 42 | request.setHouseNumber(context.queryParam("housenumber")); 43 | 44 | return request; 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /src/test/java/de/komoot/photon/nominatim/testdb/CollectingImporter.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.nominatim.testdb; 2 | 3 | import org.assertj.core.api.ListAssert; 4 | import org.assertj.core.api.ObjectAssert; 5 | import de.komoot.photon.Importer; 6 | import de.komoot.photon.PhotonDoc; 7 | 8 | import java.util.AbstractList; 9 | import java.util.ArrayList; 10 | import java.util.List; 11 | 12 | import static org.assertj.core.api.Assertions.*; 13 | 14 | public class CollectingImporter extends AbstractList implements Importer { 15 | private final List docs = new ArrayList<>(); 16 | private int finishCalled = 0; 17 | 18 | 19 | @Override 20 | public void add(Iterable inputDocs) 21 | { 22 | for (var doc : inputDocs) { 23 | docs.add(doc); 24 | } 25 | } 26 | 27 | @Override 28 | public void finish() { 29 | ++finishCalled; 30 | } 31 | 32 | @Override 33 | public int size() { 34 | return docs.size(); 35 | } 36 | 37 | @Override 38 | public PhotonDoc get(int idx) { 39 | return docs.get(idx); 40 | } 41 | 42 | public int getFinishCalled() { 43 | return finishCalled; 44 | } 45 | 46 | public ObjectAssert assertThatByPlaceId(long placeId) { 47 | return assertThat(docs.stream().filter(d -> d.getPlaceId() == placeId)) 48 | .hasSize(1) 49 | .first(); 50 | } 51 | 52 | public ObjectAssert assertThatByRow(PlacexTestRow row) { 53 | return assertThatByPlaceId(row.getPlaceId()) 54 | .satisfies(row::assertEquals); 55 | } 56 | 57 | public ListAssert assertThatAllByRow(OsmlineTestRow row) { 58 | return assertThat(docs.stream().filter(d -> d.getPlaceId().equals(row.getPlaceId()))) 59 | .isNotEmpty() 60 | .allSatisfy(row::assertEquals); 61 | } 62 | 63 | public ListAssert assertThatAllByRow(PlacexTestRow row) { 64 | return assertThat(docs.stream().filter(d -> d.getPlaceId().equals(row.getPlaceId()))) 65 | .isNotEmpty() 66 | .allSatisfy(row::assertEquals); 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /src/test/java/de/komoot/photon/nominatim/testdb/OsmlineTestRow.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.nominatim.testdb; 2 | 3 | import de.komoot.photon.PhotonDoc; 4 | import org.junit.jupiter.api.Assertions; 5 | import org.springframework.jdbc.core.JdbcTemplate; 6 | 7 | public class OsmlineTestRow { 8 | private static long placeIdSequence = 100000; 9 | 10 | private final Long placeId; 11 | private Long parentPlaceId; 12 | private Long osmId = 23L; 13 | private Integer startnumber; 14 | private Integer endnumber; 15 | private Integer step; 16 | private String countryCode = "de"; 17 | private String lineGeo; 18 | 19 | public OsmlineTestRow() { 20 | placeId = placeIdSequence++; 21 | lineGeo = "LINESTRING(0 0, 0.1 0.1, 0 0.2)"; 22 | } 23 | 24 | public OsmlineTestRow number(int start, int end, int step) { 25 | startnumber = start; 26 | endnumber = end; 27 | this.step = step; 28 | return this; 29 | } 30 | 31 | public OsmlineTestRow parent(PlacexTestRow row) { 32 | parentPlaceId = row.getPlaceId(); 33 | return this; 34 | } 35 | 36 | public OsmlineTestRow geom(String geom) { 37 | lineGeo = geom; 38 | return this; 39 | } 40 | 41 | public OsmlineTestRow add(JdbcTemplate jdbc) { 42 | jdbc.update("INSERT INTO location_property_osmline (place_id, parent_place_id, osm_id," 43 | + " startnumber, endnumber, step, linegeo, country_code, indexed_status)" 44 | + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0)", 45 | placeId, parentPlaceId, osmId, startnumber, endnumber, step, lineGeo, countryCode); 46 | 47 | return this; 48 | } 49 | 50 | public void assertEquals(PhotonDoc doc) { 51 | Assertions.assertEquals("W", doc.getOsmType()); 52 | Assertions.assertEquals(osmId, (Long) doc.getOsmId()); 53 | Assertions.assertEquals("place", doc.getTagKey()); 54 | Assertions.assertEquals("house_number", doc.getTagValue()); 55 | Assertions.assertEquals(30, (Integer) doc.getRankAddress()); 56 | } 57 | 58 | public Long getPlaceId() { 59 | return this.placeId; 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/json/NominatimDumpHeader.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.json; 2 | 3 | import com.fasterxml.jackson.annotation.JsonAnySetter; 4 | import com.fasterxml.jackson.annotation.JsonProperty; 5 | import de.komoot.photon.UsageException; 6 | import org.apache.logging.log4j.LogManager; 7 | import org.apache.logging.log4j.Logger; 8 | 9 | import java.util.Date; 10 | import java.util.HashMap; 11 | import java.util.Map; 12 | 13 | public class NominatimDumpHeader { 14 | private static final Logger LOGGER = LogManager.getLogger(); 15 | 16 | public static final String DOCUMENT_TYPE = "NominatimDumpFile"; 17 | public static final String EXPECTED_VERSION = "0.1.0"; 18 | 19 | private String generator; 20 | private Date dataTimestamp; 21 | private NominatimDumpFileFeatures features; 22 | private Map extraProperties = new HashMap<>(); 23 | 24 | @JsonProperty(DumpFields.HEADER_VERSION) 25 | void setVersion(String version) { 26 | if (!EXPECTED_VERSION.equals(version)) { 27 | LOGGER.error("Dump file header has version '{}'. Expect version '{}'", 28 | version, EXPECTED_VERSION); 29 | throw new UsageException("Invalid dump file."); 30 | } 31 | } 32 | 33 | @JsonProperty(DumpFields.HEADER_GENERATOR) 34 | void setGenerator(String generator) { 35 | this.generator = generator; 36 | } 37 | 38 | @JsonProperty(DumpFields.HEADER_DB_TIME) 39 | void setDataTimestamp(Date timestamp) { 40 | dataTimestamp = timestamp; 41 | } 42 | 43 | @JsonProperty(DumpFields.HEADER_FEATURES) 44 | void setFeatures(NominatimDumpFileFeatures features) { this.features = features; } 45 | 46 | public Date getDataTimestamp() { 47 | return dataTimestamp; 48 | } 49 | 50 | @JsonAnySetter 51 | void setExtraProperties(String key, String value) { 52 | extraProperties.put(key, value); 53 | } 54 | 55 | public boolean isSortedByCountry() { 56 | return features != null && features.isSortedByCountry; 57 | } 58 | 59 | public boolean hasAddressLines() { 60 | return features == null || features.hasAddressLines; 61 | } 62 | 63 | } 64 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/nominatim/model/AddressRow.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.nominatim.model; 2 | 3 | import java.util.Map; 4 | 5 | /** 6 | * Representation of an address as returned by Nominatim's get_addressdata PL/pgSQL function. 7 | */ 8 | public class AddressRow { 9 | private final NameMap name; 10 | private final ContextMap context; 11 | private final AddressType addressType; 12 | private final boolean isPostCode; 13 | 14 | private AddressRow(NameMap name, ContextMap context, AddressType addressType, boolean isPostCode) { 15 | this.name = name; 16 | this.context = context; 17 | this.addressType = addressType; 18 | this.isPostCode = isPostCode; 19 | } 20 | 21 | public static AddressRow make(Map name, String osmKey, String osmValue, 22 | int rankAddress, String[] languages) { 23 | ContextMap context = new ContextMap(); 24 | 25 | if (("place".equals(osmKey) && "postcode".equals(osmValue)) 26 | || ("boundary".equals(osmKey) && "postal_code".equals(osmValue))) { 27 | return new AddressRow( 28 | new NameMap().setName("ref", name, "ref"), 29 | context, 30 | AddressType.fromRank(rankAddress), 31 | true 32 | ); 33 | } else { 34 | // Makes US state abbreviations searchable. 35 | context.addName("default", name.get("ISO3166-2")); 36 | } 37 | 38 | return new AddressRow( 39 | new NameMap().setLocaleNames(name, languages), 40 | context, 41 | AddressType.fromRank(rankAddress), 42 | false); 43 | } 44 | 45 | public AddressType getAddressType() { 46 | return addressType; 47 | } 48 | 49 | public boolean isPostcode() { 50 | return isPostCode; 51 | } 52 | 53 | public boolean isUsefulForContext() { 54 | return !name.isEmpty() && !isPostcode(); 55 | } 56 | 57 | public NameMap getName() { 58 | return this.name; 59 | } 60 | 61 | public ContextMap getContext() { 62 | return context; 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/test/java/de/komoot/photon/api/ApiBaseTester.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.api; 2 | 3 | import de.komoot.photon.App; 4 | import de.komoot.photon.ESBaseTester; 5 | 6 | import java.io.BufferedReader; 7 | import java.io.IOException; 8 | import java.io.InputStreamReader; 9 | import java.net.HttpURLConnection; 10 | import java.net.URI; 11 | import java.nio.file.Path; 12 | import java.util.Arrays; 13 | import java.util.stream.Collectors; 14 | import java.util.stream.Stream; 15 | 16 | import static org.assertj.core.api.Assertions.assertThatIOException; 17 | 18 | public class ApiBaseTester extends ESBaseTester { 19 | private static final int LISTEN_PORT = 30234; 20 | private String photonDirectory; 21 | 22 | @Override 23 | public void setUpES(Path dataDirectory) throws IOException { 24 | super.setUpES(dataDirectory); 25 | photonDirectory = dataDirectory.toString(); 26 | } 27 | 28 | protected void startAPI(String... extraParams) throws Exception { 29 | // Get the actual port of the test OpenSearch instance 30 | String testPort = getTestServer().getHttpPort(); 31 | 32 | final String[] params = Stream.concat( 33 | Stream.of("-cluster", TEST_CLUSTER_NAME, 34 | "-listen-port", Integer.toString(LISTEN_PORT), 35 | "-transport-addresses", "127.0.0.1:" + testPort, 36 | "-data-dir", photonDirectory), 37 | Arrays.stream(extraParams)).toArray(String[]::new); 38 | 39 | App.main(params); 40 | } 41 | 42 | protected HttpURLConnection connect(String url) throws IOException { 43 | String urlString = "http://127.0.0.1:" + LISTEN_PORT + url.replace(" ", "%20"); 44 | return (HttpURLConnection) URI.create(urlString).toURL().openConnection(); 45 | } 46 | 47 | protected String readURL(String url) throws IOException { 48 | return new BufferedReader(new InputStreamReader(connect(url).getInputStream())) 49 | .lines().collect(Collectors.joining("\n")); 50 | } 51 | 52 | protected void assertHttpError(String url, int expectedCode) { 53 | assertThatIOException() 54 | .isThrownBy(() -> readURL(url)) 55 | .withMessageContaining("response code: " + expectedCode); 56 | 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/GenericSearchHandler.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon; 2 | 3 | import de.komoot.photon.query.RequestBase; 4 | import de.komoot.photon.query.RequestFactory; 5 | import de.komoot.photon.searcher.ResultFormatter; 6 | import de.komoot.photon.searcher.SearchHandler; 7 | import de.komoot.photon.searcher.StreetDupesRemover; 8 | import io.javalin.http.Context; 9 | import io.javalin.http.Handler; 10 | import org.jetbrains.annotations.NotNull; 11 | 12 | import java.io.IOException; 13 | 14 | public class GenericSearchHandler implements Handler { 15 | private final RequestFactory requestFactory; 16 | private final SearchHandler requestHandler; 17 | private final ResultFormatter formatter; 18 | 19 | public GenericSearchHandler(RequestFactory requestFactory, SearchHandler requestHandler, 20 | ResultFormatter formatter) { 21 | this.requestFactory = requestFactory; 22 | this.requestHandler = requestHandler; 23 | this.formatter = formatter; 24 | } 25 | 26 | @Override 27 | public void handle(@NotNull Context context) { 28 | final T searchRequest = requestFactory.create(context); 29 | 30 | var results = requestHandler.search(searchRequest); 31 | 32 | // Further filtering 33 | if (searchRequest.getDedupe()){ 34 | results = new StreetDupesRemover(searchRequest.getLanguage()).execute(results); 35 | } 36 | 37 | // Restrict to the requested limit. 38 | if (results.size() > searchRequest.getLimit()) { 39 | results = results.subList(0, searchRequest.getLimit()); 40 | } 41 | 42 | String debugInfo = null; 43 | if (searchRequest.getDebug()) { 44 | debugInfo = requestHandler.dumpQuery(searchRequest); 45 | } 46 | 47 | try { 48 | context.status(200) 49 | .result(formatter.convert( 50 | results, searchRequest.getLanguage(), 51 | searchRequest.getReturnGeometry(), 52 | searchRequest.getDebug(), debugInfo)); 53 | } catch (IOException e) { 54 | context.status(400) 55 | .result("{\"message\": \"Error creating json.\"}"); 56 | } 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/nominatim/model/AddressType.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.nominatim.model; 2 | 3 | import java.util.Arrays; 4 | import java.util.List; 5 | import java.util.stream.Collectors; 6 | 7 | /** 8 | * List of address ranks available to Photon. 9 | *

10 | * The different types correspond to the address parts available in GeocodeJSON. This type also defines 11 | * the mapping toward Nominatim's address ranks. 12 | */ 13 | public enum AddressType { 14 | HOUSE("house", 29, 30, 3), 15 | STREET("street", 26, 28, 2), 16 | LOCALITY("locality", 22, 25, 1), 17 | DISTRICT("district", 17, 21, 1), 18 | CITY("city", 13, 16, 3), 19 | COUNTY("county", 10, 12, 1), 20 | STATE("state", 5, 9, 1), 21 | COUNTRY("country", 4, 4, 2), 22 | OTHER("other", 0, 0, 1); 23 | 24 | private final String name; 25 | private final int minRank; 26 | private final int maxRank; 27 | private final int searchPrio; 28 | 29 | AddressType(String name, int minRank, int maxRank, int searchPrio) { 30 | this.name = name; 31 | this.minRank = minRank; 32 | this.maxRank = maxRank; 33 | this.searchPrio = searchPrio; 34 | } 35 | 36 | /** 37 | * Convert a Nominatim address rank into a Photon address type. 38 | * 39 | * @param addressRank Nominatim address rank. 40 | * @return The corresponding address type or null if not covered. 41 | */ 42 | public static AddressType fromRank(int addressRank) { 43 | for (AddressType a : AddressType.values()) { 44 | if (a.coversRank(addressRank)) { 45 | return a; 46 | } 47 | } 48 | 49 | return null; 50 | } 51 | 52 | /** 53 | * Check if the given address rank is mapped to the given address type. 54 | * 55 | * @param addressRank Nominatim address rank. 56 | * @return True, if the type covers the rank. 57 | */ 58 | public boolean coversRank(int addressRank) { 59 | return addressRank >= minRank && addressRank <= maxRank; 60 | } 61 | 62 | public String getName() { 63 | return name; 64 | } 65 | 66 | public static List getNames() { 67 | return Arrays.stream(AddressType.values()).map(AddressType::getName).collect(Collectors.toList()); 68 | } 69 | 70 | public int getSearchPrio() { 71 | return searchPrio; 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/StatusRequestHandler.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon; 2 | 3 | import io.javalin.http.Context; 4 | import io.javalin.http.Handler; 5 | import org.jetbrains.annotations.NotNull; 6 | 7 | import java.io.IOException; 8 | import java.util.LinkedHashMap; 9 | import java.util.Map; 10 | import java.util.jar.Attributes; 11 | import java.util.jar.Manifest; 12 | 13 | public class StatusRequestHandler implements Handler { 14 | private final Server server; 15 | private final String version; 16 | private final String gitCommit; 17 | 18 | protected StatusRequestHandler(Server server) { 19 | this.server = server; 20 | Attributes manifestAttributes = readManifestAttributes(); 21 | this.version = getManifestAttribute(manifestAttributes, "Implementation-Version"); 22 | this.gitCommit = getManifestAttribute(manifestAttributes, "Git-Commit"); 23 | } 24 | 25 | @Override 26 | public void handle(@NotNull Context context) throws IOException { 27 | DatabaseProperties dbProperties = server.loadFromDatabase(); 28 | String importDateStr = ""; 29 | if (dbProperties.getImportDate() != null) { 30 | importDateStr = dbProperties.getImportDate().toInstant().toString(); 31 | } 32 | Map response = new LinkedHashMap<>(); 33 | response.put("status", "Ok"); 34 | response.put("import_date", importDateStr); 35 | response.put("version", version); 36 | response.put("git_commit", gitCommit); 37 | context.json(response); 38 | } 39 | 40 | private Attributes readManifestAttributes() { 41 | try { 42 | var resource = getClass().getResource("/META-INF/MANIFEST.MF"); 43 | if (resource != null) { 44 | try (var stream = resource.openStream()) { 45 | return new Manifest(stream).getMainAttributes(); 46 | } 47 | } 48 | } catch (IOException e) { 49 | // Ignore, will return unknown values 50 | } 51 | return null; 52 | } 53 | 54 | private String getManifestAttribute(Attributes attributes, String name) { 55 | if (attributes != null) { 56 | String value = attributes.getValue(name); 57 | if (value != null) { 58 | return value; 59 | } 60 | } 61 | return "unknown"; 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/test/resources/test-schema.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE placex ( 2 | place_id BIGINT NOT NULL, 3 | parent_place_id BIGINT, 4 | linked_place_id BIGINT, 5 | importance FLOAT, 6 | indexed_date TIMESTAMP, 7 | geometry_sector INTEGER, 8 | rank_address SMALLINT, 9 | rank_search SMALLINT, 10 | partition SMALLINT, 11 | indexed_status SMALLINT, 12 | osm_id int8 NOT NULL, 13 | osm_type char(1) NOT NULL, 14 | class text NOT NULL, 15 | type text NOT NULL, 16 | name JSON, 17 | admin_level smallint, 18 | address JSON, 19 | extratags JSON, 20 | geometry Geometry, 21 | wikipedia TEXT, -- calculated wikipedia article name (language:title) 22 | country_code varchar(2), 23 | housenumber TEXT, 24 | postcode TEXT, 25 | centroid GEOMETRY 26 | ); 27 | 28 | CREATE TABLE place_addressline ( 29 | place_id BIGINT, 30 | address_place_id BIGINT, 31 | distance FLOAT, 32 | cached_rank_address SMALLINT, 33 | fromarea boolean, 34 | isaddress boolean 35 | ); 36 | 37 | CREATE TABLE location_property_osmline ( 38 | place_id BIGINT NOT NULL, 39 | osm_id BIGINT, 40 | parent_place_id BIGINT, 41 | geometry_sector INTEGER, 42 | indexed_date TIMESTAMP, 43 | startnumber INTEGER, 44 | endnumber INTEGER, 45 | step SMALLINT, 46 | partition SMALLINT, 47 | indexed_status SMALLINT, 48 | linegeo GEOMETRY, 49 | address JSON, 50 | postcode TEXT, 51 | country_code VARCHAR(2) 52 | ); 53 | 54 | 55 | CREATE ALIAS ST_Envelope FOR "de.komoot.photon.nominatim.testdb.Helpers.envelope"; 56 | 57 | CREATE TABLE country_name ( 58 | country_code character varying(2), 59 | name JSON, 60 | country_default_language_code character varying(2), 61 | partition integer 62 | ); 63 | 64 | INSERT INTO country_name 65 | VALUES ('de', JSON '{"name" : "Deutschland", "name:en" : "Germany"}', 'de', 2); 66 | INSERT INTO country_name 67 | VALUES ('us', JSON '{"name" : "USA", "name:en" : "United States"}', 'en', 1); 68 | INSERT INTO country_name 69 | VALUES ('hu', JSON '{"name" : "Magyarország", "name:en" : "Hungary"}', 'hu', 12); 70 | INSERT INTO country_name 71 | VALUES ('nl', JSON '{"name" : "Nederland", "name:en" : "Netherlands"}', null, 2); 72 | 73 | 74 | CREATE TABLE photon_updates ( 75 | rel TEXT, 76 | place_id BIGINT, 77 | operation TEXT, 78 | indexed_date TIMESTAMP 79 | ); 80 | 81 | 82 | CREATE TABLE import_status ( 83 | lastimportdate timestamp with TIME ZONE NOT NULL, 84 | indexed boolean 85 | ); -------------------------------------------------------------------------------- /src/test/java/de/komoot/photon/api/ApiLanguagesTest.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.api; 2 | 3 | import de.komoot.photon.App; 4 | import org.junit.jupiter.api.AfterEach; 5 | import org.junit.jupiter.api.io.TempDir; 6 | 7 | import de.komoot.photon.Importer; 8 | import de.komoot.photon.PhotonDoc; 9 | import org.junit.jupiter.api.Test; 10 | 11 | import java.nio.file.Path; 12 | import java.util.*; 13 | 14 | import static net.javacrumbs.jsonunit.assertj.JsonAssertions.assertThatJson; 15 | 16 | class ApiLanguagesTest extends ApiBaseTester { 17 | @TempDir 18 | private Path dataDirectory; 19 | 20 | @AfterEach 21 | void shutdown() { 22 | App.shutdown(); 23 | } 24 | 25 | protected PhotonDoc createDoc(int id, String value, String... names) { 26 | return new PhotonDoc() 27 | .placeId(id).osmType("N").osmId(id).tagKey("place").tagValue(value) 28 | .centroid(makePoint(1.0, 2.34)) 29 | .names(makeDocNames(names)); 30 | } 31 | 32 | private void importPlaces(String... languages) throws Exception { 33 | getProperties().setLanguages(languages); 34 | setUpES(dataDirectory); 35 | Importer instance = makeImporter(); 36 | instance.add(List.of(createDoc(1000, "city", 37 | "name:en", "thething", "name:fr", "letruc", "name:ch", "dasding"))); 38 | instance.add(List.of(createDoc(1001, "town", 39 | "name:ch", "thething", "name:fr", "letruc", "name:en", "dasding"))); 40 | instance.finish(); 41 | refresh(); 42 | } 43 | 44 | @Test 45 | void testOnlyImportSelectedLanguages() throws Exception { 46 | importPlaces("en"); 47 | startAPI(); 48 | 49 | assertThatJson(readURL("/api?q=thething")).isObject() 50 | .node("features").isArray() 51 | .hasSize(1) 52 | .element(0).isObject() 53 | .node("properties").isObject() 54 | .containsEntry("osm_id", 1000); 55 | 56 | assertThatJson(readURL("/api?q=letruc")).isObject() 57 | .node("features").isArray() 58 | .hasSize(0); 59 | } 60 | 61 | @Test 62 | void testUseImportLanguagesWhenNoOtherIsGiven() throws Exception { 63 | importPlaces("en", "fr", "ch"); 64 | startAPI(); 65 | 66 | assertThatJson(readURL("/api?q=thething")).isObject() 67 | .node("features").isArray() 68 | .hasSize(2); 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/test/java/de/komoot/photon/nominatim/model/ContextMapTest.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.nominatim.model; 2 | 3 | import org.junit.jupiter.api.Test; 4 | 5 | import java.util.Map; 6 | import java.util.Set; 7 | 8 | import static org.junit.jupiter.api.Assertions.*; 9 | 10 | public class ContextMapTest { 11 | private ContextMap testMap() { 12 | ContextMap map = new ContextMap(); 13 | map.addName("default", "n1"); 14 | map.addName("default", "n2"); 15 | map.addName("old", "former"); 16 | return map; 17 | } 18 | 19 | @Test 20 | void testAddName() { 21 | ContextMap map = new ContextMap(); 22 | 23 | assertTrue(map.isEmpty()); 24 | 25 | map.addName("default", "something"); 26 | assertEquals(Set.of("something"), map.get("default")); 27 | 28 | map.addName("alt", "else"); 29 | assertEquals(Set.of("something"), map.get("default")); 30 | assertEquals(Set.of("else"), map.get("alt")); 31 | 32 | map.addName("default", "45"); 33 | assertEquals(Set.of("something", "45"), map.get("default")); 34 | assertEquals(Set.of("else"), map.get("alt")); 35 | 36 | map.addName("alt", "else"); 37 | assertEquals(Set.of("something", "45"), map.get("default")); 38 | assertEquals(Set.of("else"), map.get("alt")); 39 | } 40 | 41 | @Test 42 | void testAddFromSimpleMap() { 43 | ContextMap map = testMap(); 44 | 45 | map.addAll(Map.of("alt", "XX", "default", "n3", "old", "former")); 46 | 47 | assertEquals(3, map.size()); 48 | assertEquals(Set.of("n1", "n2", "n3"), map.get("default")); 49 | assertEquals(Set.of("XX"), map.get("alt")); 50 | assertEquals(Set.of("former"), map.get("old")); 51 | } 52 | 53 | @Test 54 | void testAddFromContextMap() { 55 | ContextMap map = testMap(); 56 | 57 | ContextMap other = new ContextMap(); 58 | other.addName("default", "n1"); 59 | other.addName("default", "n3"); 60 | other.addName("alt", "XX"); 61 | other.addName("alt", "YY"); 62 | 63 | map.addAll(other); 64 | assertEquals(3, map.size()); 65 | assertEquals(Set.of("n1", "n2", "n3"), map.get("default")); 66 | assertEquals(Set.of("XX", "YY"), map.get("alt")); 67 | assertEquals(Set.of("former"), map.get("old")); 68 | 69 | map.addName("alt", "ZZ"); 70 | assertEquals(Set.of("XX", "YY", "ZZ"), map.get("alt")); 71 | assertEquals(Set.of("XX", "YY"), other.get("alt")); 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/query/StructuredSearchRequest.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.query; 2 | 3 | public class StructuredSearchRequest extends SearchRequestBase { 4 | private String countryCode; 5 | private String state; 6 | private String county; 7 | private String city; 8 | private String postCode; 9 | private String district; 10 | private String street; 11 | private String houseNumber; 12 | 13 | public String getCounty() { 14 | return county; 15 | } 16 | 17 | public void setCounty(String county) { 18 | this.county = county; 19 | } 20 | 21 | public String getCity() { 22 | return city; 23 | } 24 | 25 | public void setCity(String city) { 26 | this.city = city; 27 | } 28 | 29 | public String getPostCode() { 30 | return postCode; 31 | } 32 | 33 | public void setPostCode(String postCode) { 34 | this.postCode = postCode; 35 | } 36 | 37 | public String getDistrict() { 38 | return district; 39 | } 40 | 41 | public void setDistrict(String district) { 42 | this.district = district; 43 | } 44 | 45 | public String getStreet() { 46 | return street; 47 | } 48 | 49 | public void setStreet(String street) { 50 | this.street = street; 51 | } 52 | 53 | public String getHouseNumber() { 54 | return houseNumber; 55 | } 56 | 57 | public void setHouseNumber(String houseNumber) { 58 | this.houseNumber = houseNumber; 59 | } 60 | 61 | public String getCountryCode() { 62 | return countryCode; 63 | } 64 | 65 | public void setCountryCode(String countryCode) { 66 | this.countryCode = countryCode; 67 | } 68 | 69 | public String getState() { 70 | return state; 71 | } 72 | 73 | public void setState(String state) { 74 | this.state = state; 75 | } 76 | 77 | public boolean hasState() { return state != null && !state.isBlank(); } 78 | 79 | public boolean hasDistrict() { return district != null && !district.isBlank(); } 80 | 81 | public boolean hasPostCode() { return postCode != null && !postCode.isBlank(); } 82 | 83 | public boolean hasCityOrPostCode() { return (city != null && !city.isBlank()) || hasPostCode(); } 84 | 85 | public boolean hasCounty() { return county != null && !county.isBlank(); } 86 | 87 | public boolean hasStreet() { return (street != null && !street.isBlank()) || hasHouseNumber(); } 88 | 89 | public boolean hasHouseNumber() { return houseNumber != null && !houseNumber.isBlank(); } 90 | } 91 | -------------------------------------------------------------------------------- /src/test/java/de/komoot/photon/query/QueryGeometryTest.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.query; 2 | 3 | import de.komoot.photon.ESBaseTester; 4 | import de.komoot.photon.Importer; 5 | import de.komoot.photon.PhotonDoc; 6 | import de.komoot.photon.searcher.PhotonResult; 7 | import org.junit.jupiter.api.BeforeEach; 8 | import org.junit.jupiter.api.Test; 9 | import org.junit.jupiter.api.io.TempDir; 10 | 11 | import java.io.IOException; 12 | import java.nio.file.Path; 13 | import java.util.List; 14 | 15 | import static org.assertj.core.api.Assertions.*; 16 | import static net.javacrumbs.jsonunit.assertj.JsonAssertions.assertThatJson; 17 | 18 | class QueryGeometryTest extends ESBaseTester { 19 | private int testDocId = 10000; 20 | 21 | @BeforeEach 22 | void setup(@TempDir Path dataDirectory) throws IOException { 23 | getProperties().setSupportGeometries(true); 24 | setUpES(dataDirectory); 25 | } 26 | 27 | private PhotonDoc createDoc(String geometry) { 28 | ++testDocId; 29 | return new PhotonDoc() 30 | .placeId(testDocId).osmType("N").osmId(testDocId).tagKey("place").tagValue("city") 31 | .names(makeDocNames("name", "Muffle Flu")) 32 | .geometry(makeDocGeometry(geometry)) 33 | .centroid(makePoint(1.0, 2.34)); 34 | } 35 | 36 | private List search() { 37 | final var request = new SimpleSearchRequest(); 38 | request.setQuery("muffle flu"); 39 | 40 | return getServer().createSearchHandler(1).search(request); 41 | } 42 | 43 | 44 | @Test 45 | void testSearchGetPolygon() { 46 | Importer instance = makeImporter(); 47 | instance.add(List.of(createDoc("POLYGON ((30 10, 40 40, 20 40, 10 20, 30 10))"))); 48 | instance.finish(); 49 | refresh(); 50 | 51 | assertThat(search()) 52 | .element(0) 53 | .satisfies(p -> 54 | assertThatJson(p.getGeometry()).isObject() 55 | .containsEntry("type", "Polygon")); 56 | } 57 | 58 | @Test 59 | void testSearchGetLineString() { 60 | Importer instance = makeImporter(); 61 | instance.add(List.of(createDoc("LINESTRING (30 10, 10 30, 40 40)"))); 62 | instance.finish(); 63 | refresh(); 64 | 65 | assertThat(search()) 66 | .element(0) 67 | .satisfies(p -> 68 | assertThatJson(p.getGeometry()).isObject() 69 | .containsEntry("type", "LineString")); 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/nominatim/PostgisDataAdapter.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.nominatim; 2 | 3 | import net.postgis.jdbc.PGgeometry; 4 | import org.apache.logging.log4j.LogManager; 5 | import org.apache.logging.log4j.Logger; 6 | import org.locationtech.jts.geom.Geometry; 7 | import org.locationtech.jts.io.ParseException; 8 | import org.locationtech.jts.io.WKTReader; 9 | import org.springframework.jdbc.core.JdbcTemplate; 10 | 11 | import java.sql.ResultSet; 12 | import java.sql.SQLException; 13 | import java.util.HashMap; 14 | import java.util.Map; 15 | 16 | /** 17 | * Utility functions to parse data from and create SQL for PostgreSQL/PostGIS. 18 | */ 19 | public class PostgisDataAdapter implements DBDataAdapter { 20 | private static final Logger LOGGER = LogManager.getLogger(); 21 | 22 | @Override 23 | public Map getMap(ResultSet rs, String columnName) throws SQLException { 24 | Map map = (Map) rs.getObject(columnName); 25 | if (map == null) { 26 | return new HashMap<>(); 27 | } 28 | 29 | return map; 30 | } 31 | 32 | @Override 33 | public Geometry extractGeometry(ResultSet rs, String columnName) throws SQLException { 34 | PGgeometry wkt = (PGgeometry) rs.getObject(columnName); 35 | if (wkt != null) { 36 | try { 37 | StringBuffer sb = new StringBuffer(); 38 | wkt.getGeometry().outerWKT(sb); 39 | 40 | Geometry geometry = new WKTReader().read(sb.toString()); 41 | geometry.setSRID(4326); 42 | return geometry; 43 | } catch (ParseException e) { 44 | // ignore 45 | LOGGER.error("Cannot parse database geometry", e); 46 | } 47 | } 48 | 49 | return null; 50 | } 51 | 52 | @Override 53 | public boolean hasColumn(JdbcTemplate template, String table, String column) { 54 | return template.query("SELECT count(*) FROM information_schema.columns WHERE table_name = ? and column_name = ?", 55 | (ResultSet resultSet, int i) -> resultSet.getInt(1) > 0, 56 | table, column).getFirst(); 57 | } 58 | 59 | @Override 60 | public String deleteReturning(String deleteSQL, String columns) { 61 | return deleteSQL + " RETURNING " + columns; 62 | } 63 | 64 | @Override 65 | public String jsonArrayFromSelect(String valueSQL, String fromSQL) { 66 | return "(SELECT json_agg(val) FROM (SELECT " + valueSQL + " as val " + fromSQL + ") xxx)"; 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/opensearch/OpenSearchResult.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.opensearch; 2 | 3 | import de.komoot.photon.searcher.PhotonResult; 4 | 5 | import java.util.Map; 6 | 7 | public class OpenSearchResult implements PhotonResult { 8 | private static final String[] NAME_PRECEDENCE = {"default", "housename", "int", "loc", "reg", "alt", "old"}; 9 | 10 | private double score = 0.0; 11 | private final double[] extent; 12 | private final double[] coordinates; 13 | private final String geometry; 14 | private final Map infos; 15 | private final Map> localeTags; 16 | 17 | OpenSearchResult(double[] extent, double[] coordinates, Map infos, Map> localeTags, String geometry) { 18 | this.extent = extent; 19 | this.coordinates = coordinates; 20 | this.infos = infos; 21 | this.localeTags = localeTags; 22 | this.geometry = geometry; 23 | } 24 | 25 | public OpenSearchResult setScore(double score) { 26 | this.score = score; 27 | return this; 28 | } 29 | 30 | @Override 31 | public Object get(String key) { 32 | return infos.get(key); 33 | } 34 | 35 | @Override 36 | public String getLocalised(String key, String language) { 37 | final var map = getMap(key); 38 | if (map == null) return null; 39 | 40 | if (map.get(language) != null) { 41 | // language specific field 42 | return map.get(language); 43 | } 44 | 45 | if ("name".equals(key)) { 46 | for (String name : NAME_PRECEDENCE) { 47 | if (map.containsKey(name)) 48 | return map.get(name); 49 | } 50 | } 51 | 52 | return map.get("default"); 53 | } 54 | 55 | @Override 56 | public Map getMap(String key) { 57 | return localeTags.get(key); 58 | } 59 | 60 | @Override 61 | public double[] getCoordinates() { 62 | return coordinates; 63 | } 64 | 65 | public String getGeometry() { 66 | return geometry; 67 | } 68 | 69 | @Override 70 | public double[] getExtent() { 71 | return extent; 72 | } 73 | 74 | @Override 75 | public double getScore() { 76 | return score; 77 | } 78 | 79 | @Override 80 | public Map getRawData() { 81 | return Map.of( 82 | "score", score, 83 | "infos", infos, 84 | "localeTags", localeTags); 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /src/test/java/de/komoot/photon/api/ApiMetricsTest.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.api; 2 | 3 | import de.komoot.photon.App; 4 | import org.junit.jupiter.api.AfterAll; 5 | import org.junit.jupiter.api.AfterEach; 6 | import org.junit.jupiter.api.BeforeAll; 7 | import org.junit.jupiter.api.Test; 8 | import org.junit.jupiter.api.TestInstance; 9 | import org.junit.jupiter.api.io.TempDir; 10 | 11 | import java.nio.file.Path; 12 | 13 | import static org.assertj.core.api.Assertions.assertThat; 14 | 15 | @TestInstance(TestInstance.Lifecycle.PER_CLASS) 16 | class ApiMetricsTest extends ApiBaseTester { 17 | 18 | @BeforeAll 19 | void setUp(@TempDir Path dataDirectory) throws Exception { 20 | setUpES(dataDirectory); 21 | refresh(); 22 | } 23 | 24 | @AfterEach 25 | void shutdown() { 26 | App.shutdown(); 27 | } 28 | 29 | @AfterAll 30 | @Override 31 | public void tearDown() { 32 | shutdownES(); 33 | } 34 | 35 | @Test 36 | void testMetricsEndpointReturnsOpenSearchMetrics() throws Exception { 37 | startAPI("-metrics-enable", "prometheus"); 38 | 39 | String metrics = readURL("/metrics"); 40 | 41 | // Check for OpenSearch index metrics 42 | assertThat(metrics).contains("opensearch_documents_count{"); 43 | assertThat(metrics).contains("opensearch_index_size_bytes{"); 44 | assertThat(metrics).contains("opensearch_search{"); 45 | assertThat(metrics).contains("opensearch_search_time_millis_milliseconds{"); 46 | assertThat(metrics).contains("opensearch_indexing{"); 47 | assertThat(metrics).contains("opensearch_indexing_time_millis_milliseconds{"); 48 | 49 | // Check for OpenSearch cluster metrics 50 | assertThat(metrics).contains("opensearch_cluster_shards_active{"); 51 | assertThat(metrics).contains("opensearch_cluster_shards_relocating{"); 52 | assertThat(metrics).contains("opensearch_cluster_shards_unassigned{"); 53 | assertThat(metrics).contains("opensearch_cluster_health_status{"); 54 | 55 | // Check for index tag on index metrics 56 | assertThat(metrics).contains("index=\"photon\""); 57 | 58 | // Check for JVM metrics 59 | assertThat(metrics).contains("jvm_memory"); 60 | assertThat(metrics).contains("jvm_gc"); 61 | assertThat(metrics).contains("jvm_threads"); 62 | } 63 | 64 | @Test 65 | void testMetricsEndpointReturns404WhenDisabled() throws Exception { 66 | startAPI(); 67 | 68 | var conn = connect("/metrics"); 69 | assertThat(conn.getResponseCode()).isEqualTo(404); 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /.github/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Submit a new issue only if you are sure it is a missing feature or a bug. Otherwise please discuss the topic in the [mailing list](https://lists.openstreetmap.org/listinfo/photon) first. 2 | 3 | ## We love pull requests. Here's a quick guide: 4 | 5 | 1. [Fork the repo](https://help.github.com/articles/fork-a-repo) and create a branch for your new feature or bug fix. 6 | 7 | 2. Run the tests. We only take pull requests with passing tests: `mvn clean test` 8 | 9 | 3. Add at least one test for your change. Only refactoring and documentation changes 10 | require no new tests. Also make sure you submit a change specific to exactly one issue. If you have ideas for multiple 11 | changes please create separate pull requests. 12 | 13 | 4. Make the test(s) pass. 14 | 15 | 5. Push to your fork and [submit a pull request](https://help.github.com/articles/using-pull-requests). A button should 16 | appear on your fork its github page afterwards. 17 | 18 | ## Code formatting 19 | 20 | We use IntelliJ defaults and a very similar configuration for NetBeans defined in the root pom.xml. For eclipse there is this [configuration](https://github.com/graphhopper/graphhopper/files/481920/GraphHopper.Formatter.zip). Also for other IDEs 21 | it should be simple to match: 22 | 23 | * Java indent is 4 spaces 24 | * Line width is 100 characters 25 | * The rest is left to Java coding standards but disable "auto-format on save" to prevent unnecessary format changes. 26 | * Currently we do not care about import section that much, avoid changing it 27 | * Unix line endings (should be handled via git) 28 | 29 | And in case we didn't emphasize it enough: we love tests! 30 | 31 | ## Changes to mapping and index settings 32 | 33 | It is possible to change the mapping layout or index settings any time as 34 | long as they are compatible with the current layout. Photon reapplies the 35 | mapping and index settings on startup to make sure it conforms to the latest 36 | code. 37 | 38 | **Warning:** the kind of modifications that can be done on an existing 39 | index are limited. Always test if your modifications are compatible by importing 40 | a database with the version before your changes and then running Photon with 41 | the version with your changes applied. 42 | 43 | If the mappings or the settings are changed in an incompatible way that 44 | requires a reimport, then you must increase the database version in 45 | `src/main/java/de/komoot/photon/elasticsearch/DatabaseProperties.java`. 46 | For major/minor/patch always use the version of the next release. If the 47 | version number already points to the next release, increase the dev-version. 48 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/ConfigExtraTags.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon; 2 | 3 | import com.fasterxml.jackson.core.JsonGenerator; 4 | 5 | import java.io.IOException; 6 | import java.util.Arrays; 7 | import java.util.HashMap; 8 | import java.util.List; 9 | import java.util.Map; 10 | import java.util.stream.Collectors; 11 | 12 | public class ConfigExtraTags { 13 | 14 | private final boolean allowAll; 15 | private final String[] tags; 16 | 17 | public ConfigExtraTags() { 18 | allowAll = false; 19 | tags = new String[0]; 20 | } 21 | 22 | public ConfigExtraTags(List tags) { 23 | this.allowAll = tags.size() == 1 && "ALL".equals(tags.getFirst()); 24 | this.tags = allowAll ? null : tags.toArray(new String[0]); 25 | } 26 | 27 | public void writeFilteredExtraTags(JsonGenerator writer, String fieldName, Map sourceTags) throws IOException { 28 | if (!sourceTags.isEmpty()) { 29 | if (allowAll) { 30 | writer.writeObjectField(fieldName, sourceTags); 31 | } else if (tags.length > 0) { 32 | boolean foundTag = false; 33 | 34 | for (String tag : tags) { 35 | String value = sourceTags.get(tag); 36 | if (value != null) { 37 | if (!foundTag) { 38 | writer.writeObjectFieldStart(fieldName); 39 | foundTag = true; 40 | } 41 | writer.writeStringField(tag, value); 42 | } 43 | } 44 | 45 | if (foundTag) { 46 | writer.writeEndObject(); 47 | } 48 | } 49 | } 50 | } 51 | 52 | public Map filterExtraTags(Map sourceTags) { 53 | if (allowAll || sourceTags.isEmpty()) { 54 | return sourceTags; 55 | } 56 | 57 | if (tags.length == 0) { 58 | return Map.of(); 59 | } 60 | 61 | final Map newMap = new HashMap<>(); 62 | for (var key : tags) { 63 | final var value = sourceTags.get(key); 64 | if (value != null) { 65 | newMap.put(key, value); 66 | } 67 | } 68 | 69 | return newMap; 70 | } 71 | 72 | public List asConfigParam() { 73 | if (allowAll) { 74 | return List.of("ALL"); 75 | } 76 | 77 | return Arrays.stream(tags).collect(Collectors.toList()); 78 | } 79 | 80 | @Override 81 | public String toString() { 82 | return allowAll ? "" : Arrays.toString(tags); 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/searcher/StreetDupesRemover.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.searcher; 2 | 3 | import de.komoot.photon.Constants; 4 | 5 | import java.util.ArrayList; 6 | import java.util.HashSet; 7 | import java.util.List; 8 | 9 | /** 10 | * Filter out duplicate streets from the list. 11 | */ 12 | public class StreetDupesRemover { 13 | private final String language; 14 | 15 | public StreetDupesRemover(String language) { 16 | this.language = language; 17 | } 18 | 19 | public List execute(List results) { 20 | final List filteredItems = new ArrayList<>(results.size()); 21 | final HashSet keys = new HashSet<>(); 22 | 23 | for (PhotonResult result : results) { 24 | if ("highway".equals(result.get(Constants.OSM_KEY))) { 25 | // result is a street 26 | final String postcode = (String) result.get(Constants.POSTCODE); 27 | final String name = result.getLocalised(Constants.NAME, language); 28 | 29 | if (postcode != null && name != null) { 30 | // street has a postcode and name 31 | 32 | // OSM_VALUE is part of key to avoid deduplication of e.g. bus_stops and streets with same name 33 | String key = (String) result.get(Constants.OSM_VALUE); 34 | if (key == null) { 35 | key = ""; 36 | } 37 | 38 | if (language.equals("nl")) { 39 | final String onlyDigitsPostcode = stripNonDigits(postcode); 40 | key += ":" + onlyDigitsPostcode + ":" + name; 41 | } else { 42 | key += ":" + postcode + ":" + name; 43 | } 44 | 45 | if (keys.contains(key)) { 46 | // an osm highway object (e.g. street or bus_stop) with this osm_value + name + postcode is already part of the result list 47 | continue; 48 | } 49 | keys.add(key); 50 | } 51 | } 52 | filteredItems.add(result); 53 | } 54 | 55 | return filteredItems; 56 | } 57 | 58 | private static String stripNonDigits( 59 | final CharSequence input /* inspired by seh's comment */) { 60 | final StringBuilder sb = new StringBuilder( 61 | input.length() /* also inspired by seh's comment */); 62 | for (int i = 0; i < input.length(); i++) { 63 | final char c = input.charAt(i); 64 | if (c > 47 && c < 58) { 65 | sb.append(c); 66 | } 67 | } 68 | return sb.toString(); 69 | } 70 | 71 | } 72 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/searcher/TagFilter.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.searcher; 2 | 3 | import java.util.Objects; 4 | 5 | /** 6 | * Filter description for a single filter by OSM key or key/value. 7 | */ 8 | public record TagFilter(TagFilterKind kind, String key, String value) { 9 | 10 | public boolean isKeyOnly() { 11 | return value == null; 12 | } 13 | 14 | public boolean isValueOnly() { 15 | return key == null; 16 | } 17 | 18 | /** 19 | * Create a new tag filter from a osm-tag filter description. 20 | * 21 | * @param filter Tag filter description. 22 | * @return The appropriate tag filter object or null if the filter string has an invalid format. 23 | */ 24 | public static TagFilter buildOsmTagFilter(String filter) { 25 | TagFilterKind kind = null; 26 | String key = null; 27 | String value = null; 28 | 29 | String[] parts = filter.split(":"); 30 | 31 | if (parts.length == 2) { 32 | boolean excludeKey = parts[0].startsWith("!"); 33 | boolean excludeValue = parts[1].startsWith("!"); 34 | 35 | key = (excludeKey ? parts[0].substring(1) : parts[0]).trim(); 36 | if (key.isEmpty()) { 37 | key = null; 38 | } 39 | value = (excludeValue ? parts[1].substring(1) : parts[1]).trim(); 40 | 41 | if (!value.isEmpty()) { 42 | if (key != null && !excludeKey && excludeValue) { 43 | kind = TagFilterKind.EXCLUDE_VALUE; 44 | } else { 45 | kind = excludeKey || excludeValue ? TagFilterKind.EXCLUDE : TagFilterKind.INCLUDE; 46 | } 47 | } 48 | } else if (parts.length == 1 && parts[0].equals(filter)) { 49 | boolean exclude = filter.startsWith("!"); 50 | 51 | key = exclude ? filter.substring(1) : filter; 52 | 53 | if (!key.isEmpty()) { 54 | kind = exclude ? TagFilterKind.EXCLUDE : TagFilterKind.INCLUDE; 55 | } 56 | } 57 | 58 | return (kind == null) ? null : new TagFilter(kind, key, value); 59 | } 60 | 61 | @Override 62 | public String toString() { 63 | return "TagFilter{" + 64 | "kind=" + kind + 65 | ", key='" + key + '\'' + 66 | ", value='" + value + '\'' + 67 | '}'; 68 | } 69 | 70 | @Override 71 | public boolean equals(Object o) { 72 | if (this == o) return true; 73 | if (o == null || getClass() != o.getClass()) return false; 74 | TagFilter tagFilter = (TagFilter) o; 75 | return kind == tagFilter.kind && Objects.equals(key, tagFilter.key) && Objects.equals(value, tagFilter.value); 76 | } 77 | 78 | } 79 | -------------------------------------------------------------------------------- /src/test/java/de/komoot/photon/api/ApiDedupeTest.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.api; 2 | 3 | import static net.javacrumbs.jsonunit.assertj.JsonAssertions.assertThatJson; 4 | 5 | import de.komoot.photon.App; 6 | import de.komoot.photon.Importer; 7 | import de.komoot.photon.PhotonDoc; 8 | import java.nio.file.Path; 9 | import java.util.List; 10 | import org.junit.jupiter.api.*; 11 | import org.junit.jupiter.api.io.TempDir; 12 | import org.junit.jupiter.params.ParameterizedTest; 13 | import org.junit.jupiter.params.provider.ValueSource; 14 | 15 | /** 16 | * Tests for dedupe works correctly 17 | */ 18 | @TestInstance(TestInstance.Lifecycle.PER_CLASS) 19 | class ApiDedupeTest extends ApiBaseTester { 20 | 21 | @BeforeAll 22 | void setUp(@TempDir Path dataDirectory) throws Exception { 23 | setUpES(dataDirectory); 24 | Importer instance = makeImporter(); 25 | 26 | instance.add( 27 | List.of( 28 | new PhotonDoc() 29 | .placeId(1000) 30 | .osmType("W") 31 | .osmId(1000) 32 | .tagKey("highway") 33 | .tagValue("residential") 34 | .postcode("1000") 35 | .rankAddress(26) 36 | .centroid(makePoint(15.94174, 45.80355)) 37 | .names(makeDocNames("name", "Pfanove")) 38 | ) 39 | ); 40 | instance.add( 41 | List.of( 42 | new PhotonDoc() 43 | .placeId(1001) 44 | .osmType("W") 45 | .osmId(1001) 46 | .tagKey("highway") 47 | .tagValue("residential") 48 | .postcode("1000") 49 | .rankAddress(26) 50 | .centroid(makePoint(15.94192, 45.802429)) 51 | .names(makeDocNames("name", "Pfanove")) 52 | ) 53 | ); 54 | 55 | instance.finish(); 56 | refresh(); 57 | startAPI(); 58 | } 59 | 60 | @AfterAll 61 | public void tearDown() { 62 | App.shutdown(); 63 | shutdownES(); 64 | } 65 | 66 | @ParameterizedTest 67 | @ValueSource( 68 | strings = { 69 | "/api?q=Pfanove", // basic search 70 | "/api?q=Pfanove&dedupe=1", // explicitly enabled dedupe 71 | } 72 | ) 73 | void testEnabledDedupe(String baseUrl) throws Exception { 74 | assertThatJson(readURL(baseUrl)).isObject().node("features").isArray().hasSize(1); 75 | } 76 | 77 | @ParameterizedTest 78 | @ValueSource(strings = { "/api?q=Pfanove&dedupe=0" }) 79 | void testDisabledDedupe(String baseUrl) throws Exception { 80 | assertThatJson(readURL(baseUrl)).isObject().node("features").isArray().hasSize(2); 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/query/RequestBase.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.query; 2 | 3 | import de.komoot.photon.searcher.TagFilter; 4 | 5 | import java.util.*; 6 | 7 | public class RequestBase { 8 | private String language = "default"; 9 | private int limit = 15; 10 | private boolean debug = false; 11 | private boolean dedupe = true; 12 | private boolean returnGeometry = false; 13 | 14 | private final List osmTagFilters = new ArrayList<>(1); 15 | private final Set layerFilters = new HashSet<>(1); 16 | private final Set includeCategories = new HashSet<>(); 17 | private final Set excludeCategories = new HashSet<>(); 18 | 19 | public String getLanguage() { 20 | return language; 21 | } 22 | 23 | public int getLimit() { 24 | return limit; 25 | } 26 | 27 | public boolean getDebug() { 28 | return debug; 29 | } 30 | 31 | public boolean getDedupe() { 32 | return dedupe; 33 | } 34 | 35 | public boolean getReturnGeometry() { 36 | return returnGeometry; 37 | } 38 | 39 | public List getOsmTagFilters() { 40 | return osmTagFilters; 41 | } 42 | 43 | public Set getLayerFilters() { 44 | return layerFilters; 45 | } 46 | 47 | public Set getIncludeCategories() { 48 | return includeCategories; 49 | } 50 | 51 | public Set getExcludeCategories() { 52 | return excludeCategories; 53 | } 54 | 55 | public void setLanguage(String language) { 56 | if (language != null) { 57 | this.language = language; 58 | } 59 | } 60 | 61 | public void setLimit(Integer limit, int maxLimit) { 62 | if (limit != null) { 63 | this.limit = Integer.max(1, Integer.min(maxLimit, limit)); 64 | } 65 | } 66 | 67 | public void setDebug(Boolean debug) { 68 | if (debug != null) { 69 | this.debug = debug; 70 | } 71 | } 72 | 73 | public void setDedupe(Boolean dedupe) { 74 | if (dedupe != null) { 75 | this.dedupe = dedupe; 76 | } 77 | } 78 | 79 | public void setReturnGeometry(Boolean returnGeometry) { 80 | if (returnGeometry != null) { 81 | this.returnGeometry = returnGeometry; 82 | } 83 | } 84 | 85 | void addOsmTagFilter(TagFilter filter) { 86 | osmTagFilters.add(filter); 87 | } 88 | 89 | void addLayerFilters(Collection filters) { 90 | layerFilters.addAll(filters); 91 | } 92 | 93 | void addIncludeCategories(Collection categories) { 94 | includeCategories.addAll(categories); 95 | } 96 | 97 | void addExcludeCategories(Collection categories) { 98 | excludeCategories.addAll(categories); 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/nominatim/NominatimConnector.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.nominatim; 2 | 3 | import de.komoot.photon.DatabaseProperties; 4 | import de.komoot.photon.nominatim.model.AddressRow; 5 | import de.komoot.photon.nominatim.model.NameMap; 6 | import org.apache.commons.dbcp2.BasicDataSource; 7 | import org.springframework.jdbc.core.JdbcTemplate; 8 | import org.springframework.jdbc.datasource.DataSourceTransactionManager; 9 | import org.springframework.transaction.support.TransactionTemplate; 10 | 11 | import java.util.Date; 12 | import java.util.HashMap; 13 | import java.util.List; 14 | import java.util.Map; 15 | 16 | /** 17 | * Base class for workers connecting to a Nominatim database 18 | */ 19 | public class NominatimConnector { 20 | protected final DBDataAdapter dbutils; 21 | protected final DatabaseProperties dbProperties; 22 | protected final JdbcTemplate template; 23 | protected final TransactionTemplate txTemplate; 24 | protected Map countryNames; 25 | 26 | protected NominatimConnector(String host, int port, String database, String username, String password, DBDataAdapter dataAdapter, DatabaseProperties dbProperties) { 27 | BasicDataSource dataSource = new BasicDataSource(); 28 | 29 | dataSource.setUrl(String.format("jdbc:postgresql://%s:%d/%s", host, port, database)); 30 | dataSource.setUsername(username); 31 | if (password != null) { 32 | dataSource.setPassword(password); 33 | } 34 | 35 | // Keep disabled or server-side cursors won't work. 36 | dataSource.setDefaultAutoCommit(false); 37 | 38 | txTemplate = new TransactionTemplate(new DataSourceTransactionManager(dataSource)); 39 | 40 | template = new JdbcTemplate(dataSource); 41 | template.setFetchSize(100000); 42 | 43 | dbutils = dataAdapter; 44 | this.dbProperties = dbProperties; 45 | } 46 | 47 | public Date getLastImportDate() { 48 | List importDates = template.query( 49 | "SELECT lastimportdate FROM import_status ORDER BY lastimportdate DESC LIMIT 1", 50 | (rs, rowNum) -> rs.getTimestamp("lastimportdate")); 51 | 52 | return importDates.isEmpty() ? null : importDates.getFirst(); 53 | } 54 | 55 | public Map loadCountryNames(String[] languages) { 56 | if (countryNames == null) { 57 | countryNames = new HashMap<>(); 58 | // Default for places outside any country. 59 | countryNames.put("", new NameMap()); 60 | template.query("SELECT country_code, name FROM country_name", rs -> { 61 | var names = AddressRow.make(dbutils.getMap(rs, "name"), 62 | "place", 63 | "country", 64 | 4, 65 | languages).getName(); 66 | if (!names.isEmpty()) { 67 | countryNames.put(rs.getString("country_code"), names); 68 | } 69 | }); 70 | } 71 | 72 | return countryNames; 73 | } 74 | 75 | } 76 | -------------------------------------------------------------------------------- /src/test/java/de/komoot/photon/query/QueryReverseTest.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.query; 2 | 3 | import de.komoot.photon.ESBaseTester; 4 | import de.komoot.photon.PhotonDoc; 5 | import org.locationtech.jts.geom.Coordinate; 6 | import de.komoot.photon.Importer; 7 | import de.komoot.photon.searcher.PhotonResult; 8 | import org.junit.jupiter.api.AfterAll; 9 | import org.junit.jupiter.api.BeforeAll; 10 | import org.junit.jupiter.api.Test; 11 | import org.junit.jupiter.api.TestInstance; 12 | import org.junit.jupiter.api.io.TempDir; 13 | import org.junit.jupiter.params.ParameterizedTest; 14 | import org.junit.jupiter.params.provider.ValueSource; 15 | import org.locationtech.jts.geom.Point; 16 | 17 | import java.io.IOException; 18 | import java.nio.file.Path; 19 | import java.util.List; 20 | 21 | import static org.assertj.core.api.Assertions.*; 22 | 23 | @TestInstance(TestInstance.Lifecycle.PER_CLASS) 24 | class QueryReverseTest extends ESBaseTester { 25 | final Point[] TEST_POINTS = { 26 | makePoint(10, 10), 27 | makePoint(10, 10.1), 28 | makePoint(10, 10.2), 29 | makePoint(-10, -10) 30 | }; 31 | 32 | @BeforeAll 33 | void setup(@TempDir Path dataDirectory) throws IOException { 34 | setUpES(dataDirectory); 35 | 36 | Importer instance = makeImporter(); 37 | 38 | int id = 100; 39 | for (Point pt : TEST_POINTS) { 40 | instance.add(List.of(new PhotonDoc() 41 | .placeId(id).osmType("N").osmId(id++).tagKey("place").tagValue("house") 42 | .centroid(pt) 43 | .names(makeDocNames("name", "some house")) 44 | )); 45 | } 46 | 47 | instance.finish(); 48 | refresh(); 49 | } 50 | 51 | @AfterAll 52 | @Override 53 | public void tearDown() { 54 | super.tearDown(); 55 | } 56 | 57 | private List reverse(double lon, double lat, double radius, Integer limit) { 58 | final var request = new ReverseRequest(); 59 | request.setLocation(FACTORY.createPoint(new Coordinate(lon, lat))); 60 | request.setRadius(radius); 61 | if (limit != null) { 62 | request.setLimit(limit, limit); 63 | } 64 | 65 | return getServer().createReverseHandler(1).search(request); 66 | } 67 | 68 | @Test 69 | void testReverse() { 70 | assertThat(reverse(10, 10, 0.1, 1)) 71 | .satisfiesExactly(p -> assertThat(p.get("osm_id")).isEqualTo(100)); 72 | } 73 | 74 | @Test 75 | void testDefaultLimitIsOne() { 76 | assertThat(reverse(10, 10, 20, null)) 77 | .satisfiesExactly( 78 | p -> assertThat(p.get("osm_id")).isEqualTo(100)); 79 | } 80 | 81 | @ParameterizedTest 82 | @ValueSource(ints = {2, 3, 10}) 83 | void testReverseMultiple(int limit) { 84 | assertThat(reverse(10, 10, 20, limit)) 85 | .satisfiesExactly( 86 | p -> assertThat(p.get("osm_id")).isEqualTo(100), 87 | p -> assertThat(p.get("osm_id")).isEqualTo(101)); 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /docs/categories.md: -------------------------------------------------------------------------------- 1 | # Categories 2 | 3 | Categories allow to define custom filtering on top of a Photon geo database. 4 | Each place in the Photon database can be assigned an arbitrary number of 5 | categories. Then the "include" and "exclude" parameter can be used to filter 6 | results for presence or absence of certain categories. 7 | 8 | ## Category definition 9 | 10 | A category name consists of a sequence of labels which are separated by dots. 11 | A label is an arbitrary string consisting of letters, numbers, underscore or 12 | dash (or to be precise: `[a-zA-Z0-9_-]`). Category names are case-sensitive. 13 | 14 | The leading label defines the _category group_, subsequent labels the value 15 | within the group. Note that this means that your category must have at least 16 | two components: the group and one value label. 17 | 18 | The labels are considered to be forming a hierarchy. A 19 | place can then be filtered by any part of the hierarchy. A depth between 20 | 1 and 4 labels is supported for filtering. 21 | 22 | ## Adding categories to a place 23 | 24 | Categories can be added to a place via the JSON import. Add a field `categories` 25 | containing an array of all categories, you want to add. Category names that 26 | do not confirm to the category syntax are silently dropped. 27 | 28 | The Nominatim database exporter creates exactly one category entry using 29 | the (reserved) group `osm`. It contains the main tag key and value, the 30 | same as the API returns in the result as `osm_key` and `osm_value`. _Note that 31 | the exporter will replace key and value, when they do not conform to the 32 | category syntax with `place` and `yes` respectively._ 33 | 34 | ## Difference between categories and extra tags 35 | 36 | Photon supports another field for custom values: `extra`. Extra values are 37 | saved in the database as is and returned with the response. They are not 38 | indexed and cannot be searched. 39 | 40 | Categories, on the other hand, are _only_ usable for filtering. They are not 41 | saved as raw data and not returned to the user. 42 | 43 | This gives you fine-grained control over which extra information to return to 44 | the user and what filters to support. If you have data which you want to return 45 | _and_ want to filter by, simply add it twice: once in extra and again as a 46 | category. 47 | 48 | ## Filtering with `include` and `exclude` 49 | 50 | Searches and reverse searches can be filtered by category using the 51 | `include` and `exclude` parameters. 52 | 53 | The include parameter takes a comma-separated list of categories and will 54 | include a place if any of the categories is present. The include parameter 55 | may be repeated. The conditions must then be all fulfilled. 56 | 57 | The exclude parameter also takes a comma-separated list and will exclude and 58 | object when **all** categories are present. When repeated, the object will be 59 | excluded, when any of the exclude conditions is met. 60 | 61 | You can have partial label-paths in your filter condition. `include=food.shop` 62 | will match `food.shop.supermarket` and `food.shop.convenience_store` as well. 63 | However, you cannot filter by just the category group. A search with 64 | `include=food` will return an error. 65 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/opensearch/OpenSearchReverseHandler.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.opensearch; 2 | 3 | import de.komoot.photon.query.ReverseRequest; 4 | import de.komoot.photon.searcher.PhotonResult; 5 | import de.komoot.photon.searcher.SearchHandler; 6 | import org.locationtech.jts.geom.Point; 7 | import org.opensearch.client.opensearch.OpenSearchClient; 8 | import org.opensearch.client.opensearch._types.SearchType; 9 | import org.opensearch.client.opensearch._types.SortOrder; 10 | import org.opensearch.client.opensearch._types.query_dsl.Query; 11 | import org.opensearch.client.opensearch.core.SearchResponse; 12 | import org.opensearch.client.opensearch.core.search.Hit; 13 | 14 | import java.io.IOException; 15 | import java.util.List; 16 | import java.util.stream.Collectors; 17 | 18 | public class OpenSearchReverseHandler implements SearchHandler { 19 | private final OpenSearchClient client; 20 | private final String queryTimeout; 21 | 22 | public OpenSearchReverseHandler(OpenSearchClient client, int queryTimeoutSec) { 23 | this.client = client; 24 | queryTimeout = queryTimeoutSec + "s"; 25 | } 26 | 27 | @Override 28 | public List search(ReverseRequest request) { 29 | final var queryBuilder = new ReverseQueryBuilder(request.getLocation(), request.getRadius()); 30 | queryBuilder.addQueryFilter(request.getQueryStringFilter()); 31 | queryBuilder.addLayerFilter(request.getLayerFilters()); 32 | queryBuilder.addOsmTagFilter(request.getOsmTagFilters()); 33 | queryBuilder.includeCategories(request.getIncludeCategories()); 34 | queryBuilder.excludeCategories(request.getExcludeCategories()); 35 | 36 | final var results = search(queryBuilder.build(), 37 | request.getLimit(), 38 | request.getLocationDistanceSort() ? request.getLocation() : null); 39 | 40 | return results.hits().hits().stream() 41 | .map(Hit::source) 42 | .collect(Collectors.toList()); 43 | } 44 | 45 | @Override 46 | public String dumpQuery(ReverseRequest photonRequest) { 47 | return "{}"; 48 | } 49 | 50 | private SearchResponse search(Query query, int limit, Point location) { 51 | try { 52 | return client.search(s -> { 53 | s.index(PhotonIndex.NAME) 54 | .searchType(SearchType.QueryThenFetch) 55 | .query(query) 56 | .size(limit) 57 | .timeout(queryTimeout); 58 | 59 | if (location != null) { 60 | s.sort(sq -> sq 61 | .geoDistance(gd -> gd 62 | .field("coordinate") 63 | .location(l -> l.latlon(ll -> ll.lat(location.getY()).lon(location.getX()))) 64 | .order(SortOrder.Asc))); 65 | } 66 | return s; 67 | }, OpenSearchResult.class); 68 | } catch (IOException e) { 69 | throw new RuntimeException("IO error during search", e); 70 | } 71 | } 72 | 73 | } 74 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/opensearch/OpenSearchSearchHandler.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.opensearch; 2 | 3 | import de.komoot.photon.query.SimpleSearchRequest; 4 | import de.komoot.photon.searcher.PhotonResult; 5 | import de.komoot.photon.searcher.SearchHandler; 6 | import org.opensearch.client.opensearch.OpenSearchClient; 7 | import org.opensearch.client.opensearch._types.SearchType; 8 | import org.opensearch.client.opensearch._types.query_dsl.Query; 9 | import org.opensearch.client.opensearch.core.SearchResponse; 10 | 11 | import java.io.IOException; 12 | import java.util.ArrayList; 13 | import java.util.List; 14 | 15 | public class OpenSearchSearchHandler implements SearchHandler { 16 | private final OpenSearchClient client; 17 | private final String queryTimeout; 18 | 19 | public OpenSearchSearchHandler(OpenSearchClient client, int queryTimeout) { 20 | this.client = client; 21 | this.queryTimeout = queryTimeout + "s"; 22 | } 23 | 24 | @Override 25 | public List search(SimpleSearchRequest request) { 26 | final int limit = request.getLimit(); 27 | final int extLimit = limit > 1 ? (int) Math.round(limit * 1.5) : 1; 28 | 29 | var results = sendQuery(buildQuery(request, false), extLimit); 30 | 31 | if (results.hits().hits().isEmpty()) { 32 | results = sendQuery(buildQuery(request, true), extLimit); 33 | } 34 | 35 | List ret = new ArrayList<>(); 36 | for (var hit : results.hits().hits()) { 37 | var score = hit.score(); 38 | var source = hit.source(); 39 | if (source != null) { 40 | if (score != null) { 41 | source.setScore(score); 42 | } 43 | ret.add(source); 44 | } 45 | } 46 | 47 | return ret; 48 | } 49 | 50 | @Override 51 | public String dumpQuery(SimpleSearchRequest simpleSearchRequest) { 52 | return "{}"; 53 | } 54 | 55 | private Query buildQuery(SimpleSearchRequest request, boolean lenient) { 56 | final var query = new SearchQueryBuilder(request.getQuery(), lenient, request.getSuggestAddresses()); 57 | query.addOsmTagFilter(request.getOsmTagFilters()); 58 | query.addLayerFilter(request.getLayerFilters()); 59 | query.addLocationBias(request.getLocationForBias(), request.getScaleForBias(), request.getZoomForBias()); 60 | query.includeCategories(request.getIncludeCategories()); 61 | query.excludeCategories(request.getExcludeCategories()); 62 | query.addBoundingBox(request.getBbox()); 63 | 64 | return query.build(); 65 | } 66 | 67 | private SearchResponse sendQuery(Query query, int limit) { 68 | try { 69 | return client.search(s -> s 70 | .index(PhotonIndex.NAME) 71 | .searchType(SearchType.QueryThenFetch) 72 | .query(query) 73 | .size(limit) 74 | .timeout(queryTimeout), OpenSearchResult.class); 75 | } catch (IOException e) { 76 | throw new RuntimeException("IO error during search", e); 77 | } 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | @rem SPDX-License-Identifier: Apache-2.0 17 | @rem 18 | 19 | @if "%DEBUG%"=="" @echo off 20 | @rem ########################################################################## 21 | @rem 22 | @rem Gradle startup script for Windows 23 | @rem 24 | @rem ########################################################################## 25 | 26 | @rem Set local scope for the variables with windows NT shell 27 | if "%OS%"=="Windows_NT" setlocal 28 | 29 | set DIRNAME=%~dp0 30 | if "%DIRNAME%"=="" set DIRNAME=. 31 | @rem This is normally unused 32 | set APP_BASE_NAME=%~n0 33 | set APP_HOME=%DIRNAME% 34 | 35 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 36 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 37 | 38 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 39 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 40 | 41 | @rem Find java.exe 42 | if defined JAVA_HOME goto findJavaFromJavaHome 43 | 44 | set JAVA_EXE=java.exe 45 | %JAVA_EXE% -version >NUL 2>&1 46 | if %ERRORLEVEL% equ 0 goto execute 47 | 48 | echo. 1>&2 49 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 50 | echo. 1>&2 51 | echo Please set the JAVA_HOME variable in your environment to match the 1>&2 52 | echo location of your Java installation. 1>&2 53 | 54 | goto fail 55 | 56 | :findJavaFromJavaHome 57 | set JAVA_HOME=%JAVA_HOME:"=% 58 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 59 | 60 | if exist "%JAVA_EXE%" goto execute 61 | 62 | echo. 1>&2 63 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 64 | echo. 1>&2 65 | echo Please set the JAVA_HOME variable in your environment to match the 1>&2 66 | echo location of your Java installation. 1>&2 67 | 68 | goto fail 69 | 70 | :execute 71 | @rem Setup the command line 72 | 73 | 74 | 75 | @rem Execute Gradle 76 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %* 77 | 78 | :end 79 | @rem End local scope for the variables with windows NT shell 80 | if %ERRORLEVEL% equ 0 goto mainEnd 81 | 82 | :fail 83 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 84 | rem the _cmd.exe /c_ return code! 85 | set EXIT_CODE=%ERRORLEVEL% 86 | if %EXIT_CODE% equ 0 set EXIT_CODE=1 87 | if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% 88 | exit /b %EXIT_CODE% 89 | 90 | :mainEnd 91 | if "%OS%"=="Windows_NT" endlocal 92 | 93 | :omega 94 | -------------------------------------------------------------------------------- /src/test/java/de/komoot/photon/query/QueryFilterLayerTest.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.query; 2 | 3 | import de.komoot.photon.ESBaseTester; 4 | import de.komoot.photon.Importer; 5 | import de.komoot.photon.PhotonDoc; 6 | import de.komoot.photon.searcher.PhotonResult; 7 | import org.junit.jupiter.api.*; 8 | import org.junit.jupiter.api.io.TempDir; 9 | import org.locationtech.jts.geom.Coordinate; 10 | 11 | import java.nio.file.Path; 12 | import java.util.Arrays; 13 | import java.util.List; 14 | import java.util.stream.Collectors; 15 | 16 | import static org.assertj.core.api.Assertions.*; 17 | 18 | @TestInstance(TestInstance.Lifecycle.PER_CLASS) 19 | class QueryFilterLayerTest extends ESBaseTester { 20 | @BeforeAll 21 | void setUp(@TempDir Path dataDirectory) throws Exception { 22 | setUpES(dataDirectory); 23 | Importer instance = makeImporter(); 24 | 25 | int id = 0; 26 | 27 | int[] docRanks = {10, 13, 14, 22}; // state, city * 2, locality 28 | for (int rank : docRanks) { 29 | instance.add(List.of(new PhotonDoc() 30 | .placeId(id).osmType("W").osmId(++id).tagKey("place").tagValue("value") 31 | .names(makeDocNames("name", "berlin")) 32 | .centroid(makePoint(10, 10)) 33 | 34 | .rankAddress(rank))); 35 | } 36 | 37 | instance.finish(); 38 | refresh(); 39 | } 40 | 41 | @AfterAll 42 | @Override 43 | public void tearDown() { 44 | super.tearDown(); 45 | } 46 | 47 | private List searchWithLayers(String... layers) { 48 | SimpleSearchRequest request = new SimpleSearchRequest(); 49 | request.setQuery("berlin"); 50 | request.addLayerFilters(Arrays.stream(layers).collect(Collectors.toSet())); 51 | 52 | return getServer().createSearchHandler(1).search(request); 53 | } 54 | 55 | private List reverse(String... layers) { 56 | ReverseRequest request = new ReverseRequest(); 57 | request.setLocation(FACTORY.createPoint(new Coordinate(10, 10))); 58 | request.setLimit(15, 15); 59 | request.addLayerFilters(Arrays.stream(layers).collect(Collectors.toSet())); 60 | 61 | return getServer().createReverseHandler(1).search(request); 62 | } 63 | 64 | @Test 65 | void testSearchSingleLayer() { 66 | assertThat(searchWithLayers("city")) 67 | .hasSize(2) 68 | .allSatisfy(p -> assertThat(p.get("type")).isEqualTo("city")); 69 | } 70 | 71 | @Test 72 | void testSearchMultipleLayers() { 73 | assertThat(searchWithLayers("city", "locality")) 74 | .hasSize(3) 75 | .allSatisfy(p -> assertThat(p.get("type")).isNotEqualTo("state")); 76 | } 77 | 78 | @Test 79 | void testReverseSingleLayer() { 80 | assertThat(reverse("city")) 81 | .hasSize(2) 82 | .allSatisfy(p -> assertThat(p.get("type")).isEqualTo("city")); 83 | } 84 | 85 | @Test 86 | void testReverseMultipleLayers() { 87 | assertThat(reverse("city", "locality")) 88 | .hasSize(3) 89 | .allSatisfy(p -> assertThat(p.get("type")).isNotEqualTo("state")); 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/opensearch/OsmTagFilter.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.opensearch; 2 | 3 | import de.komoot.photon.searcher.TagFilter; 4 | import de.komoot.photon.searcher.TagFilterKind; 5 | import org.opensearch.client.opensearch._types.FieldValue; 6 | import org.opensearch.client.opensearch._types.query_dsl.BoolQuery; 7 | import org.opensearch.client.opensearch._types.query_dsl.Query; 8 | import org.opensearch.client.opensearch._types.query_dsl.TermsQuery; 9 | 10 | import java.util.Collections; 11 | import java.util.List; 12 | 13 | public class OsmTagFilter { 14 | private BoolQuery.Builder includeTagQueryBuilder = null; 15 | private BoolQuery.Builder excludeTagQueryBuilder = null; 16 | 17 | public OsmTagFilter withOsmTagFilters(List filters) { 18 | for (var filter : filters) { 19 | addOsmTagFilter(filter); 20 | } 21 | return this; 22 | } 23 | 24 | public Query build() { 25 | if (includeTagQueryBuilder != null || excludeTagQueryBuilder != null) { 26 | return BoolQuery.of(q -> { 27 | if (includeTagQueryBuilder != null) { 28 | q.must(includeTagQueryBuilder.build().toQuery()); 29 | } 30 | if (excludeTagQueryBuilder != null) { 31 | q.mustNot(excludeTagQueryBuilder.build().toQuery()); 32 | } 33 | return q; 34 | }).toQuery(); 35 | } 36 | 37 | return null; 38 | } 39 | 40 | private void addOsmTagFilter(TagFilter filter) { 41 | if (filter.kind() == TagFilterKind.EXCLUDE_VALUE) { 42 | appendIncludeTerm(BoolQuery.of(q -> q 43 | .must(makeTermsQuery("osm_key", filter.key())) 44 | .mustNot(makeTermsQuery("osm_value", filter.value()))).toQuery()); 45 | } else { 46 | Query query; 47 | if (filter.isKeyOnly()) { 48 | query = makeTermsQuery("osm_key", filter.key()); 49 | } else if (filter.isValueOnly()) { 50 | query = makeTermsQuery("osm_value", filter.value()); 51 | } else { 52 | query = BoolQuery.of(q -> q 53 | .must(makeTermsQuery("osm_key", filter.key())) 54 | .must(makeTermsQuery("osm_value", filter.value()))).toQuery(); 55 | } 56 | 57 | if (filter.kind() == TagFilterKind.INCLUDE) { 58 | appendIncludeTerm(query); 59 | } else { 60 | appendExcludeTerm(query); 61 | } 62 | } 63 | } 64 | 65 | private void appendIncludeTerm(Query query) { 66 | if (includeTagQueryBuilder == null) { 67 | includeTagQueryBuilder = new BoolQuery.Builder(); 68 | } 69 | 70 | includeTagQueryBuilder.should(query); 71 | } 72 | 73 | private void appendExcludeTerm(Query query) { 74 | if (excludeTagQueryBuilder == null) { 75 | excludeTagQueryBuilder = new BoolQuery.Builder(); 76 | } 77 | 78 | excludeTagQueryBuilder.should(query); 79 | } 80 | 81 | private static Query makeTermsQuery(String field, String term) { 82 | return TermsQuery.of(q -> q 83 | .field(field) 84 | .terms(t -> t.value(Collections.singletonList(FieldValue.of(term))))).toQuery(); 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /docs/synonyms.md: -------------------------------------------------------------------------------- 1 | # Using Synonyms and Classification Terms 2 | 3 | Photon has built-in support for using custom query-time synonyms and 4 | special phrases for searching a place by its type. This document explains 5 | how to configure this feature. 6 | 7 | ## Configuration 8 | 9 | Synonyms and classification terms are configured with a JSON file which can 10 | be added to a Photon server instance using the command line parameter 11 | `-synonym-file`. Synonyms are a run-time feature. Handing in a synonym list 12 | at import time has no effect. The list of synonyms in use can simply be 13 | changed by restarting the Photon server with a different synonym list (or 14 | not at all, if you want to completely disable the feature again). 15 | 16 | Here is a simple example of a synonym configuration file: 17 | 18 | ``` 19 | { 20 | "search_synonyms": [ 21 | "first,1st", 22 | "second,2nd" 23 | ], 24 | "classification_terms": [ 25 | { 26 | "key": "aeroway", 27 | "value": "aerodrome", 28 | "terms": ["airport", "airfield"] 29 | }, 30 | { 31 | "key": "railway", 32 | "value": "station", 33 | "terms": ["station"] 34 | } 35 | ] 36 | } 37 | ``` 38 | 39 | The file has two main sections: `search_synonyms` allows for simple synonym 40 | replacements in the query. `classification_term` defines descriptive terms 41 | for a OSM key/value pair. 42 | 43 | ## Synonyms 44 | 45 | The `search_synonyms` section must contain a list of synonym replacements. 46 | Each entry contains a comma-separated of terms that may be replaced with each 47 | other in the query. Only single-word terms are allowed. That means the terms 48 | must neither contain spaces nor hyphens or the like.[^1] 49 | 50 | [^1] This is a restriction of ElasticSearch 5. Synonym replacement does not 51 | create correct term positions when multi-word synonyms are involved. 52 | 53 | ## Classification Terms 54 | 55 | The second section `classification_terms` defines a list of OSM key/value 56 | pairs with their descriptive terms. `place` and `building` may not be used as 57 | keys. Neither will `highway=residential` nor `highway=unclassified` work. 58 | There may be multiple entries for the same key/value pair (for example, 59 | if you have extra entries for each supported language). 60 | 61 | The classification terms can help improve search when the type of an object 62 | is used in the query but does not appear in the name. For example, with the 63 | configuration given above a query of "Berlin Station" will find a railway 64 | station which in OpenStreetMap has the name "Berlin" and also one with 65 | the name "Berlin Hauptbahnhof". 66 | 67 | Classification terms do not enable searching for objects of a certain type. 68 | "Station London" will not get you all railway stations in London but a 69 | railway station _named_ London. 70 | 71 | ## Usage Advice 72 | 73 | Use synonyms and classification terms sparingly and only if you can be 74 | reasonably sure that they will target the intended part of the address. 75 | Short or frequent terms can have unexpected side-effects and worsen the 76 | search results. For example, it might sound like a good idea to use synonyms 77 | to handle the abbreviation from 'Saint' to 'St'. The problem here is that 78 | 'St' is also used as an abbreviation for 'Street'. So all searches that 79 | involve a 'Street' will suddenly also search for places containing 'Saint'. 80 | 81 | Do not create synonyms for terms that are used as classification terms. 82 | Photon will not complain but again there might be unintended side effects. 83 | 84 | -------------------------------------------------------------------------------- /website/photon/static/img/opensource.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 18 | 20 | 39 | 46 | 47 | 49 | 50 | 52 | image/svg+xml 53 | 55 | 56 | 57 | 58 | 59 | 64 | 71 | 76 | 79 | 80 | 81 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/opensearch/Updater.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.opensearch; 2 | 3 | import de.komoot.photon.PhotonDoc; 4 | import org.apache.logging.log4j.LogManager; 5 | import org.apache.logging.log4j.Logger; 6 | import org.opensearch.client.opensearch.OpenSearchClient; 7 | import org.opensearch.client.opensearch.core.BulkRequest; 8 | 9 | import java.io.IOException; 10 | 11 | public class Updater implements de.komoot.photon.Updater { 12 | private static final Logger LOGGER = LogManager.getLogger(); 13 | 14 | private final OpenSearchClient client; 15 | private BulkRequest.Builder bulkRequest = new BulkRequest.Builder(); 16 | private int todoDocuments = 0; 17 | 18 | public Updater(OpenSearchClient client) { 19 | this.client = client; 20 | } 21 | 22 | public void addOrUpdate(Iterable docs) { 23 | Long placeID = null; 24 | int objectId = 0; 25 | 26 | for (var doc: docs) { 27 | if (objectId == 0) { 28 | placeID = doc.getPlaceId(); 29 | if (placeID == null) { 30 | throw new RuntimeException("Documents without place_id cannot be used for updates."); 31 | } 32 | } 33 | final String uid = PhotonDoc.makeUid(placeID, objectId++); 34 | 35 | bulkRequest.operations(op -> op 36 | .index(i -> i.index(PhotonIndex.NAME).id(uid).document(doc))); 37 | 38 | if (++todoDocuments > 10000) { 39 | updateDocuments(); 40 | } 41 | } 42 | 43 | if (placeID != null) { 44 | deleteSubset(placeID, objectId); 45 | } 46 | } 47 | 48 | public void delete(long placeId) { 49 | deleteSubset(placeId, 0); 50 | } 51 | 52 | private void deleteSubset(long docId, int fromObjectId) { 53 | int objectId = fromObjectId; 54 | 55 | while (exists(docId, objectId++)) { 56 | final String uid = PhotonDoc.makeUid(docId, objectId); 57 | bulkRequest.operations(op -> op 58 | .delete(d -> d.index(PhotonIndex.NAME).id(uid))); 59 | 60 | if (++todoDocuments > 10000) { 61 | updateDocuments(); 62 | } 63 | } 64 | } 65 | 66 | private boolean exists(long docId, int objectId) { 67 | try { 68 | return client.exists(e -> e.index(PhotonIndex.NAME).id(PhotonDoc.makeUid(docId, objectId))).value(); 69 | } catch (IOException e) { 70 | LOGGER.warn("IO error on exists operation", e); 71 | } 72 | return false; 73 | } 74 | 75 | @Override 76 | public void finish() { 77 | updateDocuments(); 78 | try { 79 | client.indices().refresh(r -> r.index(PhotonIndex.NAME)); 80 | } catch (IOException e) { 81 | LOGGER.warn("IO error on refresh."); 82 | } 83 | } 84 | 85 | private void updateDocuments() { 86 | if (todoDocuments > 0) { 87 | try { 88 | var response = client.bulk(bulkRequest.build()); 89 | 90 | if (response.errors()) { 91 | LOGGER.error("Errors during bulk update."); 92 | } 93 | } catch (IOException e) { 94 | LOGGER.error("IO error during bulk update", e); 95 | } 96 | 97 | bulkRequest = new BulkRequest.Builder(); 98 | todoDocuments = 0; 99 | } 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /src/test/java/de/komoot/photon/searcher/GeocodeJsonFormatterTest.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.searcher; 2 | 3 | import de.komoot.photon.Constants; 4 | import org.junit.jupiter.api.Test; 5 | 6 | import java.io.IOException; 7 | import java.util.List; 8 | 9 | import static net.javacrumbs.jsonunit.assertj.JsonAssertions.assertThatJson; 10 | 11 | class GeocodeJsonFormatterTest { 12 | 13 | @Test 14 | void testConvertPointToGeojson() throws IOException { 15 | GeocodeJsonFormatter formatter = new GeocodeJsonFormatter(); 16 | 17 | final var allPointResults = List.of( 18 | createDummyPointResult("99999", "Park Foo", "leisure", "park"), 19 | createDummyPointResult("88888", "Bar Park", "amenity", "bar")); 20 | 21 | String geojsonString = formatter.convert(allPointResults, "en", false, false, null); 22 | 23 | var features = assertThatJson(geojsonString).isObject() 24 | .containsEntry("type", "FeatureCollection") 25 | .node("features").isArray().hasSize(2); 26 | 27 | for (int i = 0; i < 2; ++i) { 28 | features.element(i).isObject() 29 | .containsEntry("type", "Feature") 30 | .node("geometry").isObject() 31 | .containsEntry("type", "Point") 32 | .node("coordinates").isArray() 33 | .isEqualTo("[42.0, 21.0]"); 34 | } 35 | 36 | features.element(0).isObject().node("properties").isObject() 37 | .containsEntry("osm_key", "leisure") 38 | .containsEntry("osm_value", "park"); 39 | features.element(1).isObject().node("properties").isObject() 40 | .containsEntry("osm_key", "amenity") 41 | .containsEntry("osm_value", "bar"); 42 | } 43 | 44 | @Test 45 | void testConvertGeometryToGeojson() throws IOException { 46 | GeocodeJsonFormatter formatter = new GeocodeJsonFormatter(); 47 | 48 | final var allResults = List.of( 49 | createDummyGeometryResult("99999", "Park Foo", "leisure", "park")); 50 | 51 | String geojsonString = formatter.convert(allResults, "en", true, false, null); 52 | 53 | assertThatJson(geojsonString).isObject() 54 | .containsEntry("type", "FeatureCollection") 55 | .node("features").isArray().hasSize(1) 56 | .element(0).isObject() 57 | .containsEntry("type", "Feature") 58 | .node("geometry").isObject() 59 | .containsEntry("type", "MultiPolygon"); 60 | } 61 | 62 | private PhotonResult createDummyPointResult(String postCode, String name, String osmKey, 63 | String osmValue) { 64 | return new MockPhotonResult() 65 | .put(Constants.POSTCODE, postCode) 66 | .putLocalized(Constants.NAME, "en", name) 67 | .put(Constants.OSM_KEY, osmKey) 68 | .put(Constants.OSM_VALUE, osmValue) 69 | .putGeometry("{\"type\":\"Point\", \"coordinates\": [42, 21]}"); 70 | } 71 | 72 | private PhotonResult createDummyGeometryResult(String postCode, String name, String osmKey, 73 | String osmValue) { 74 | return new MockPhotonResult() 75 | .put(Constants.POSTCODE, postCode) 76 | .putLocalized(Constants.NAME, "en", name) 77 | .put(Constants.OSM_KEY, osmKey) 78 | .put(Constants.OSM_VALUE, osmValue); 79 | } 80 | 81 | } 82 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/metrics/MetricsConfig.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.metrics; 2 | 3 | import de.komoot.photon.CommandLineArgs; 4 | import io.javalin.micrometer.MicrometerPlugin; 5 | import io.micrometer.core.instrument.MeterRegistry; 6 | import io.micrometer.core.instrument.binder.jvm.ClassLoaderMetrics; 7 | import io.micrometer.core.instrument.binder.jvm.JvmGcMetrics; 8 | import io.micrometer.core.instrument.binder.jvm.JvmMemoryMetrics; 9 | import io.micrometer.core.instrument.binder.jvm.JvmThreadMetrics; 10 | import io.micrometer.core.instrument.binder.system.FileDescriptorMetrics; 11 | import io.micrometer.core.instrument.binder.system.ProcessorMetrics; 12 | import io.micrometer.core.instrument.binder.system.UptimeMetrics; 13 | import io.micrometer.prometheusmetrics.PrometheusConfig; 14 | import io.micrometer.prometheusmetrics.PrometheusMeterRegistry; 15 | import org.apache.logging.log4j.LogManager; 16 | import org.apache.logging.log4j.Logger; 17 | import org.jetbrains.annotations.NotNull; 18 | import org.opensearch.client.opensearch.OpenSearchClient; 19 | 20 | public class MetricsConfig { 21 | private static final Logger LOGGER = LogManager.getLogger(); 22 | private MicrometerPlugin micrometerPlugin; 23 | private PrometheusMeterRegistry registry; 24 | private final String path = "/metrics"; 25 | 26 | private MetricsConfig() { 27 | } 28 | 29 | private void init(@NotNull OpenSearchClient client) { 30 | registry = new PrometheusMeterRegistry(PrometheusConfig.DEFAULT); 31 | registry.config().commonTags("application", "Photon"); 32 | registerJvmMetrics(registry); 33 | registerOpenSearchMetrics(client); 34 | micrometerPlugin = new MicrometerPlugin(micrometerPluginConfig -> micrometerPluginConfig.registry = registry); 35 | LOGGER.info("Metrics enabled at " + path); 36 | } 37 | 38 | private void registerJvmMetrics(MeterRegistry registry) { 39 | new ClassLoaderMetrics().bindTo(registry); 40 | new FileDescriptorMetrics().bindTo(registry); 41 | new JvmGcMetrics().bindTo(registry); 42 | new JvmMemoryMetrics().bindTo(registry); 43 | new JvmThreadMetrics().bindTo(registry); 44 | new ProcessorMetrics().bindTo(registry); 45 | new UptimeMetrics().bindTo(registry); 46 | } 47 | 48 | private void registerOpenSearchMetrics(@NotNull OpenSearchClient client) { 49 | new OpenSearchMetrics(client).bindTo(registry); 50 | } 51 | 52 | @NotNull 53 | public MicrometerPlugin getPlugin() { 54 | if (micrometerPlugin == null) { 55 | throw new IllegalStateException("MetricsConfig not initialized."); 56 | } 57 | return micrometerPlugin; 58 | } 59 | 60 | @NotNull 61 | public PrometheusMeterRegistry getRegistry() { 62 | if (registry == null) { 63 | throw new IllegalStateException("PrometheusMeterRegistry not initialized."); 64 | } 65 | return registry; 66 | } 67 | 68 | @NotNull 69 | public String getPath() { 70 | return path; 71 | } 72 | 73 | public boolean isEnabled() { 74 | return registry != null && micrometerPlugin != null; 75 | } 76 | 77 | @NotNull 78 | public static MetricsConfig setupMetrics(@NotNull CommandLineArgs args, @NotNull OpenSearchClient client) { 79 | MetricsConfig metricsConfig = new MetricsConfig(); 80 | if (args.getMetricsEnable() != null && args.getMetricsEnable().equalsIgnoreCase("prometheus")) { 81 | metricsConfig.init(client); 82 | } 83 | return metricsConfig; 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /src/test/java/de/komoot/photon/PhotonDocInterpolationSetTest.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon; 2 | 3 | import org.assertj.core.api.SoftAssertions; 4 | import org.assertj.core.data.Offset; 5 | import org.junit.jupiter.api.BeforeEach; 6 | import org.junit.jupiter.api.Test; 7 | import org.locationtech.jts.geom.Geometry; 8 | import org.locationtech.jts.io.ParseException; 9 | import org.locationtech.jts.io.WKTReader; 10 | 11 | import static org.junit.jupiter.api.Assertions.*; 12 | import static org.assertj.core.api.Assertions.*; 13 | 14 | class PhotonDocInterpolationSetTest { 15 | private final PhotonDoc baseDoc = new PhotonDoc(10000, "N", 123, "place", "house") 16 | .countryCode("de"); 17 | private final WKTReader reader = new WKTReader(); 18 | private Geometry lineGeo; 19 | 20 | private void assertCentroid(PhotonDoc doc, double x, double y) { 21 | SoftAssertions soft = new SoftAssertions(); 22 | 23 | soft.assertThat(doc.getCentroid().getX()).isEqualTo(x, Offset.offset(0.0000001)); 24 | soft.assertThat(doc.getCentroid().getY()).isEqualTo(y, Offset.offset(0.0000001)); 25 | 26 | soft.assertAll(); 27 | } 28 | 29 | private void assertDocWithHousenumber(PhotonDoc doc, String housenumber, double y) { 30 | assertAll( 31 | () -> assertNotSame(baseDoc, doc), 32 | () -> assertEquals("place", doc.getTagKey()), 33 | () -> assertEquals("house", doc.getTagValue()), 34 | () -> assertEquals(10000, doc.getPlaceId()), 35 | () -> assertEquals("N", doc.getOsmType()), 36 | () -> assertEquals(123, doc.getOsmId()), 37 | () -> assertEquals(housenumber, doc.getHouseNumber()), 38 | () -> assertCentroid(doc, 2.5, y) 39 | ); 40 | } 41 | 42 | @BeforeEach 43 | void setupGeometry() throws ParseException{ 44 | lineGeo = reader.read("LINESTRING(2.5 0.0 ,2.5 0.1)"); 45 | } 46 | 47 | @Test 48 | void testBadInterpolationReverse() { 49 | assertThat(new PhotonDocInterpolationSet(baseDoc, 34, 33, 1, lineGeo)) 50 | .isEmpty(); 51 | } 52 | 53 | @Test 54 | void testBadInterpolationLargeMulti() { 55 | assertThat(new PhotonDocInterpolationSet(baseDoc, 1, 2000, 1, lineGeo)) 56 | .isEmpty(); 57 | } 58 | 59 | @Test 60 | void testSinglePointInterpolation() { 61 | assertThat(new PhotonDocInterpolationSet(baseDoc, 2000, 2000, 1, lineGeo)) 62 | .satisfiesExactly( 63 | d -> assertThat(d) 64 | .satisfies(dh -> assertThat(dh.getHouseNumber()).isEqualTo("2000")) 65 | .satisfies(dp -> assertCentroid(dp, 2.5, 0.05))); 66 | } 67 | 68 | @Test 69 | void testSingleStepInterpolation() { 70 | assertThat(new PhotonDocInterpolationSet(baseDoc, 1, 3, 1, lineGeo)) 71 | .satisfiesExactly( 72 | d1 -> assertDocWithHousenumber(d1, "1", 0), 73 | d2 -> assertDocWithHousenumber(d2, "2", 0.05), 74 | d3 -> assertDocWithHousenumber(d3, "3", 0.1)); 75 | } 76 | 77 | @Test 78 | void testTwoStepInterpolation() { 79 | assertThat(new PhotonDocInterpolationSet(baseDoc, 16, 20, 2, lineGeo)) 80 | .satisfiesExactly( 81 | d1 -> assertDocWithHousenumber(d1, "16", 0), 82 | d2 -> assertDocWithHousenumber(d2, "18", 0.05), 83 | d3 -> assertDocWithHousenumber(d3, "20", 0.1)); 84 | } 85 | } -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/opensearch/Importer.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.opensearch; 2 | 3 | import de.komoot.photon.PhotonDoc; 4 | import org.apache.logging.log4j.LogManager; 5 | import org.apache.logging.log4j.Logger; 6 | import org.opensearch.client.opensearch.OpenSearchClient; 7 | import org.opensearch.client.opensearch.core.BulkRequest; 8 | import org.opensearch.client.opensearch.core.bulk.BulkResponseItem; 9 | 10 | import java.io.IOException; 11 | 12 | public class Importer implements de.komoot.photon.Importer { 13 | private static final Logger LOGGER = LogManager.getLogger(); 14 | 15 | private final OpenSearchClient client; 16 | private BulkRequest.Builder bulkRequest = new BulkRequest.Builder(); 17 | private int todoDocuments = 0; 18 | private boolean hasPrintedNoUpdates = false; 19 | 20 | public Importer(OpenSearchClient client) { 21 | this.client = client; 22 | } 23 | 24 | @Override 25 | public void add(Iterable docs) { 26 | Long placeID = null; 27 | int objectId = 0; 28 | for (var doc : docs) { 29 | if (objectId == 0) { 30 | placeID = doc.getPlaceId(); 31 | } 32 | if (placeID == null) { 33 | if (!hasPrintedNoUpdates) { 34 | LOGGER.warn("Documents have no place_id. Updates will not be possible."); 35 | hasPrintedNoUpdates = true; 36 | } 37 | bulkRequest.operations(op -> op 38 | .create(i -> i 39 | .index(PhotonIndex.NAME) 40 | .document(doc))); 41 | } else { 42 | final String uuid = PhotonDoc.makeUid(placeID, objectId++); 43 | bulkRequest.operations(op -> op 44 | .create(i -> i 45 | .index(PhotonIndex.NAME) 46 | .id(uuid) 47 | .document(doc))); 48 | } 49 | ++todoDocuments; 50 | 51 | if (todoDocuments % 10000 == 0) { 52 | saveDocuments(); 53 | } 54 | } 55 | } 56 | 57 | @Override 58 | public void finish() { 59 | if (todoDocuments > 0) { 60 | saveDocuments(); 61 | } 62 | 63 | try { 64 | client.indices().refresh(r -> r.index(PhotonIndex.NAME)); 65 | } catch (IOException e) { 66 | LOGGER.warn("Refresh of database failed", e); 67 | } 68 | } 69 | 70 | private void saveDocuments() { 71 | try { 72 | final var request = bulkRequest.build(); 73 | final var response = client.bulk(request); 74 | 75 | if (response.errors()) { 76 | for (BulkResponseItem bri: response.items()) { 77 | if (bri.status() != 201) { 78 | LOGGER.error("Error during bulk import: {}", bri.toJsonString()); 79 | for (var op : request.operations()) { 80 | if (op.isCreate() && bri.id() != null && bri.id().equals(op.create().id())) { 81 | LOGGER.error("Bad document: {}", op.create().document()); 82 | } 83 | } 84 | } 85 | } 86 | } 87 | } catch (IOException e) { 88 | LOGGER.error("Error during bulk import", e); 89 | } 90 | 91 | bulkRequest = new BulkRequest.Builder(); 92 | todoDocuments = 0; 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /src/test/java/de/komoot/photon/ESBaseTester.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon; 2 | 3 | import de.komoot.photon.nominatim.model.AddressRow; 4 | import de.komoot.photon.nominatim.model.NameMap; 5 | import de.komoot.photon.searcher.PhotonResult; 6 | import org.junit.jupiter.api.AfterEach; 7 | import org.locationtech.jts.geom.*; 8 | import org.locationtech.jts.io.ParseException; 9 | import org.locationtech.jts.io.WKTReader; 10 | 11 | import java.io.IOException; 12 | import java.nio.file.Path; 13 | import java.util.HashMap; 14 | import java.util.List; 15 | import java.util.Map; 16 | 17 | 18 | public class ESBaseTester { 19 | public static final String TEST_CLUSTER_NAME = TestServer.TEST_CLUSTER_NAME; 20 | protected static final GeometryFactory FACTORY = new GeometryFactory(new PrecisionModel(), 4326); 21 | 22 | private TestServer server; 23 | private final DatabaseProperties dbProperties = new DatabaseProperties(); 24 | 25 | protected NameMap makeDocNames(String... names) { 26 | Map nameMap = new HashMap<>(); 27 | 28 | for (int i = 0; i < names.length - 1; i += 2) { 29 | nameMap.put(names[i], names[i+1]); 30 | } 31 | 32 | return NameMap.makeForPlace(nameMap, dbProperties.getLanguages()); 33 | } 34 | 35 | protected Map makeAddressNames(String... names) { 36 | Map nameMap = new HashMap<>(); 37 | 38 | for (int i = 0; i < names.length - 1; i += 2) { 39 | nameMap.put(names[i], names[i+1]); 40 | } 41 | 42 | return AddressRow.make(nameMap, "place", "city", 16, dbProperties.getLanguages()).getName(); 43 | } 44 | 45 | protected Point makePoint(double x, double y) { 46 | return FACTORY.createPoint(new Coordinate(x, y)); 47 | } 48 | 49 | protected Geometry makeDocGeometry(String wkt) { 50 | try { 51 | return new WKTReader().read(wkt); 52 | } catch (ParseException e) { 53 | throw new RuntimeException(e); 54 | } 55 | } 56 | 57 | @AfterEach 58 | public void tearDown() { 59 | shutdownES(); 60 | } 61 | 62 | protected PhotonResult getById(int id) { 63 | return getById(Integer.toString(id)); 64 | } 65 | 66 | protected PhotonResult getById(String id) { 67 | return server.getByID(id); 68 | } 69 | 70 | protected List getAll() { return server.getAll(); } 71 | 72 | public void setUpES(Path dataDirectory) throws IOException { 73 | server = new TestServer(dataDirectory.toString()); 74 | server.startTestServer(TEST_CLUSTER_NAME); 75 | server.recreateIndex(dbProperties); 76 | server.refreshIndexes(); 77 | } 78 | 79 | protected Importer makeImporter() { 80 | return server.createImporter(dbProperties); 81 | } 82 | 83 | protected Updater makeUpdater() { 84 | return server.createUpdater(dbProperties); 85 | } 86 | 87 | protected Server getServer() { 88 | assert server != null; 89 | 90 | return server; 91 | } 92 | 93 | protected TestServer getTestServer() { 94 | assert server != null; 95 | 96 | return server; 97 | } 98 | 99 | protected DatabaseProperties getProperties() { 100 | return dbProperties; 101 | } 102 | 103 | protected void refresh() { 104 | server.refreshTestServer(); 105 | } 106 | 107 | /** 108 | * Shutdown the ES node 109 | */ 110 | public void shutdownES() { 111 | if (server != null) { 112 | server.stopTestServer(); 113 | } 114 | } 115 | 116 | } 117 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/opensearch/OpenSearchResultDeserializer.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.opensearch; 2 | 3 | import com.fasterxml.jackson.core.JsonParser; 4 | import com.fasterxml.jackson.databind.DeserializationContext; 5 | import com.fasterxml.jackson.databind.JsonNode; 6 | import com.fasterxml.jackson.databind.deser.std.StdDeserializer; 7 | import com.fasterxml.jackson.databind.node.ArrayNode; 8 | import com.fasterxml.jackson.databind.node.ObjectNode; 9 | import de.komoot.photon.Constants; 10 | import de.komoot.photon.searcher.PhotonResult; 11 | 12 | import java.io.IOException; 13 | import java.util.HashMap; 14 | import java.util.Map; 15 | 16 | public class OpenSearchResultDeserializer extends StdDeserializer { 17 | 18 | public OpenSearchResultDeserializer() { 19 | super(OpenSearchResult.class); 20 | } 21 | 22 | @Override 23 | public OpenSearchResult deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { 24 | final var node = (ObjectNode) p.getCodec().readTree(p); 25 | 26 | final double[] extent = extractExtent((ObjectNode) node.get("extent")); 27 | final double[] coordinates = extractCoordinate((ObjectNode) node.get("coordinate")); 28 | 29 | final Map tags = new HashMap<>(); 30 | final Map> localeTags = new HashMap<>(); 31 | 32 | String geometry = null; 33 | if (node.get("geometry") != null) { 34 | geometry = node.get("geometry").toString(); 35 | } 36 | 37 | var fieldNames = node.fieldNames(); 38 | while (fieldNames.hasNext()) { 39 | String key = fieldNames.next(); 40 | final JsonNode value = node.get(key); 41 | if (value.isTextual()) { 42 | tags.put(key, value.asText()); 43 | } else if (value.isInt()) { 44 | tags.put(key, value.asInt()); 45 | } else if (value.isLong()) { 46 | tags.put(key, value.asLong()); 47 | } else if (value.isFloatingPointNumber()) { 48 | tags.put(key, value.asDouble()); 49 | } else if (value.isObject()) { 50 | Map vtags = new HashMap<>(); 51 | ObjectNode objValue = (ObjectNode) value; 52 | var subFieldNames = objValue.fieldNames(); 53 | while (subFieldNames.hasNext()) { 54 | String subKey = subFieldNames.next(); 55 | JsonNode subValue = objValue.get(subKey); 56 | if (subValue.isTextual()) { 57 | vtags.put(subKey, subValue.asText()); 58 | } 59 | } 60 | localeTags.put(key, vtags); 61 | } 62 | } 63 | 64 | return new OpenSearchResult(extent, coordinates, tags, localeTags, geometry); 65 | } 66 | 67 | private double[] extractExtent(ObjectNode node) { 68 | if (node == null || !node.has("coordinates")) { 69 | return null; 70 | } 71 | 72 | final var coords = ((ArrayNode) node.get("coordinates")); 73 | final var nw = ((ArrayNode) coords.get(0)); 74 | final var se = ((ArrayNode) coords.get(1)); 75 | 76 | return new double[]{nw.get(0).doubleValue(), nw.get(1).doubleValue(), 77 | se.get(0).doubleValue(), se.get(1).doubleValue()}; 78 | } 79 | 80 | private double[] extractCoordinate(ObjectNode node) { 81 | if (node == null) { 82 | return PhotonResult.INVALID_COORDINATES; 83 | } 84 | 85 | return new double[]{node.get(Constants.LON).doubleValue(), node.get(Constants.LAT).doubleValue()}; 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/nominatim/model/PlaceRowMapper.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.nominatim.model; 2 | 3 | import de.komoot.photon.PhotonDoc; 4 | import de.komoot.photon.nominatim.DBDataAdapter; 5 | import org.springframework.jdbc.core.RowMapper; 6 | 7 | import java.sql.ResultSet; 8 | import java.sql.SQLException; 9 | import java.util.List; 10 | import java.util.regex.Pattern; 11 | 12 | /** 13 | * Maps the basic attributes of a placex table row to a PhotonDoc. 14 | * 15 | * This class does not complete address information (neither country information) 16 | * for the place. 17 | */ 18 | public class PlaceRowMapper implements RowMapper { 19 | private static final Pattern CATEGORY_PATTERN = Pattern.compile( 20 | String.format("[%s]+", PhotonDoc.CATEGORY_VALID_CHARS)); 21 | 22 | private final DBDataAdapter dbutils; 23 | private final String[] languages; 24 | private final boolean useGeometryColumn; 25 | 26 | public PlaceRowMapper(DBDataAdapter dbutils, String[] langauges, boolean useGeometryColumn) { 27 | this.dbutils = dbutils; 28 | this.languages = langauges; 29 | this.useGeometryColumn = useGeometryColumn; 30 | } 31 | 32 | @Override 33 | public PhotonDoc mapRow(ResultSet rs, int rowNum) throws SQLException { 34 | String osmKey = rs.getString("class"); 35 | String osmValue = rs.getString("type"); 36 | if (!CATEGORY_PATTERN.matcher(osmKey).matches()) { 37 | osmKey = "place"; 38 | osmValue = "yes"; 39 | } else if (!CATEGORY_PATTERN.matcher(osmValue).matches()) { 40 | osmValue = "yes"; 41 | } 42 | PhotonDoc doc = new PhotonDoc(rs.getLong("place_id"), 43 | rs.getString("osm_type"), rs.getLong("osm_id"), 44 | osmKey, osmValue) 45 | .names(NameMap.makeForPlace(dbutils.getMap(rs, "name"), languages)) 46 | .extraTags(dbutils.getMap(rs, "extratags")) 47 | .categories(List.of(String.format("osm.%s.%s", osmKey, osmValue))) 48 | .bbox(dbutils.extractGeometry(rs, "bbox")) 49 | .countryCode(rs.getString("country_code")) 50 | .centroid(dbutils.extractGeometry(rs, "centroid")) 51 | .rankAddress(rs.getInt("rank_address")) 52 | .postcode(rs.getString("postcode")); 53 | 54 | if (useGeometryColumn) { 55 | try { 56 | doc.geometry(dbutils.extractGeometry(rs, "geometry")); 57 | } catch (IllegalArgumentException e) { 58 | System.out.println("Could not get Geometry: " + e); 59 | } 60 | } 61 | 62 | double importance = rs.getDouble("importance"); 63 | doc.importance(rs.wasNull() ? (0.75 - rs.getInt("rank_search") / 40d) : importance); 64 | 65 | return doc; 66 | } 67 | 68 | public String makeBaseSelect() { 69 | var sql = "SELECT p.place_id, p.osm_type, p.osm_id, p.class, p.type, p.name, p.postcode," + 70 | " p.address, p.extratags, ST_Envelope(p.geometry) AS bbox," + 71 | " p.rank_address, p.rank_search, p.importance, p.country_code, p.centroid, " + 72 | dbutils.jsonArrayFromSelect( 73 | "address_place_id", 74 | "FROM place_addressline pa " + 75 | " WHERE pa.place_id IN (p.place_id, " + 76 | "coalesce(CASE WHEN p.rank_search = 30 THEN p.parent_place_id ELSE null END, p.place_id)) AND isaddress" + 77 | " ORDER BY cached_rank_address DESC") + " as addresslines"; 78 | 79 | if (useGeometryColumn) { 80 | sql += ", p.geometry"; 81 | } 82 | 83 | return sql; 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/nominatim/model/NominatimAddressCache.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.nominatim.model; 2 | 3 | import com.fasterxml.jackson.core.JsonProcessingException; 4 | import com.fasterxml.jackson.databind.ObjectMapper; 5 | import de.komoot.photon.nominatim.DBDataAdapter; 6 | import org.apache.logging.log4j.LogManager; 7 | import org.apache.logging.log4j.Logger; 8 | import org.springframework.jdbc.core.JdbcTemplate; 9 | import org.springframework.jdbc.core.RowCallbackHandler; 10 | 11 | import java.util.*; 12 | import java.util.stream.Collectors; 13 | 14 | /** 15 | * Container for caching information about address parts. 16 | */ 17 | public class NominatimAddressCache { 18 | private static final Logger LOGGER = LogManager.getLogger(); 19 | 20 | private static final String BASE_COUNTRY_QUERY = 21 | "SELECT place_id, name, class, type, rank_address FROM placex" + 22 | " WHERE rank_address between 5 and 25 AND linked_place_id is null"; 23 | 24 | private static final ObjectMapper objectMapper = new ObjectMapper(); 25 | private final Map addresses = new HashMap<>(); 26 | private final RowCallbackHandler rowMapper; 27 | 28 | public NominatimAddressCache(DBDataAdapter dbutils, String[] languages) { 29 | rowMapper = rs -> { 30 | final var row = AddressRow.make( 31 | dbutils.getMap(rs, "name"), 32 | rs.getString("class"), 33 | rs.getString("type"), 34 | rs.getInt("rank_address"), 35 | languages); 36 | if (!row.getName().isEmpty()) { 37 | addresses.put( 38 | rs.getLong("place_id"), row); 39 | } 40 | }; 41 | } 42 | 43 | public void loadCountryAddresses(JdbcTemplate template, String countryCode) { 44 | if ("".equals(countryCode)) { 45 | template.query(BASE_COUNTRY_QUERY + " AND country_code is null", rowMapper); 46 | } else { 47 | template.query(BASE_COUNTRY_QUERY + " AND country_code = ?", rowMapper, countryCode); 48 | } 49 | 50 | if (!addresses.isEmpty()) { 51 | LOGGER.info("Loaded {} address places for country {}", addresses.size(), countryCode); 52 | } 53 | } 54 | 55 | public List getOrLoadAddressList(JdbcTemplate template, String json) { 56 | if (json == null || json.isBlank()) { 57 | return List.of(); 58 | } 59 | 60 | final Long[] placeIDs = parsePlaceIdArray(json); 61 | 62 | final Long[] missing = Arrays.stream(placeIDs) 63 | .filter(id -> !addresses.containsKey(id)).toArray(Long[]::new); 64 | 65 | if (missing.length > 0) { 66 | template.query( 67 | BASE_COUNTRY_QUERY + " AND place_id = ANY(?)", 68 | rowMapper, (Object) missing); 69 | } 70 | 71 | return makeAddressList(placeIDs); 72 | } 73 | 74 | public List getAddressList(String json) { 75 | if (json == null || json.isBlank()) { 76 | return List.of(); 77 | } 78 | 79 | return makeAddressList(parsePlaceIdArray(json)); 80 | } 81 | 82 | private List makeAddressList(Long[] placeIDs) { 83 | return Arrays.stream(placeIDs) 84 | .map(addresses::get) 85 | .filter(Objects::nonNull) 86 | .collect(Collectors.toList()); 87 | } 88 | 89 | private Long[] parsePlaceIdArray(String json) { 90 | try { 91 | return objectMapper.readValue(json, Long[].class); 92 | } catch (JsonProcessingException e) { 93 | LOGGER.error("Cannot parse database response.", e); 94 | throw new RuntimeException("Parse error."); 95 | } 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /website/photon/static/js/angular-sanitize.js: -------------------------------------------------------------------------------- 1 | /* 2 | AngularJS v1.0.7 3 | (c) 2010-2012 Google, Inc. http://angularjs.org 4 | License: MIT 5 | */ 6 | (function(I,g){'use strict';function i(a){var d={},a=a.split(","),b;for(b=0;b=0;e--)if(f[e]==b)break;if(e>=0){for(c=f.length-1;c>=e;c--)d.end&&d.end(f[c]);f.length= 7 | e}}var c,h,f=[],j=a;for(f.last=function(){return f[f.length-1]};a;){h=!0;if(!f.last()||!q[f.last()]){if(a.indexOf("<\!--")===0)c=a.indexOf("--\>"),c>=0&&(d.comment&&d.comment(a.substring(4,c)),a=a.substring(c+3),h=!1);else if(B.test(a)){if(c=a.match(r))a=a.substring(c[0].length),c[0].replace(r,e),h=!1}else if(C.test(a)&&(c=a.match(s)))a=a.substring(c[0].length),c[0].replace(s,b),h=!1;h&&(c=a.indexOf("<"),h=c<0?a:a.substring(0,c),a=c<0?"":a.substring(c),d.chars&&d.chars(k(h)))}else a=a.replace(RegExp("(.*)<\\s*\\/\\s*"+ 8 | f.last()+"[^>]*>","i"),function(b,a){a=a.replace(D,"$1").replace(E,"$1");d.chars&&d.chars(k(a));return""}),e("",f.last());if(a==j)throw"Parse Error: "+a;j=a}e()}function k(a){l.innerHTML=a.replace(//g,">")}function u(a){var d=!1,b=g.bind(a,a.push);return{start:function(a,c,h){a=g.lowercase(a);!d&&q[a]&&(d=a);!d&&v[a]== 9 | !0&&(b("<"),b(a),g.forEach(c,function(a,c){var e=g.lowercase(c);if(G[e]==!0&&(w[e]!==!0||a.match(H)))b(" "),b(c),b('="'),b(t(a)),b('"')}),b(h?"/>":">"))},end:function(a){a=g.lowercase(a);!d&&v[a]==!0&&(b(""));a==d&&(d=!1)},chars:function(a){d||b(t(a))}}}var s=/^<\s*([\w:-]+)((?:\s+[\w:-]+(?:\s*=\s*(?:(?:"[^"]*")|(?:'[^']*')|[^>\s]+))?)*)\s*(\/?)\s*>/,r=/^<\s*\/\s*([\w:-]+)[^>]*>/,A=/([\w:-]+)(?:\s*=\s*(?:(?:"((?:[^"])*)")|(?:'((?:[^'])*)')|([^>\s]+)))?/g,C=/^/g, 10 | E=//g,H=/^((ftp|https?):\/\/|mailto:|#)/,F=/([^\#-~| |!])/g,p=i("area,br,col,hr,img,wbr"),x=i("colgroup,dd,dt,li,p,tbody,td,tfoot,th,thead,tr"),y=i("rp,rt"),o=g.extend({},y,x),m=g.extend({},x,i("address,article,aside,blockquote,caption,center,del,dir,div,dl,figure,figcaption,footer,h1,h2,h3,h4,h5,h6,header,hgroup,hr,ins,map,menu,nav,ol,pre,script,section,table,ul")),n=g.extend({},y,i("a,abbr,acronym,b,bdi,bdo,big,br,cite,code,del,dfn,em,font,i,img,ins,kbd,label,map,mark,q,ruby,rp,rt,s,samp,small,span,strike,strong,sub,sup,time,tt,u,var")), 11 | q=i("script,style"),v=g.extend({},p,m,n,o),w=i("background,cite,href,longdesc,src,usemap"),G=g.extend({},w,i("abbr,align,alt,axis,bgcolor,border,cellpadding,cellspacing,class,clear,color,cols,colspan,compact,coords,dir,face,headers,height,hreflang,hspace,ismap,lang,language,nohref,nowrap,rel,rev,rows,rowspan,rules,scope,scrolling,shape,span,start,summary,target,title,type,valign,value,vspace,width")),l=document.createElement("pre");g.module("ngSanitize",[]).value("$sanitize",function(a){var d=[]; 12 | z(a,u(d));return d.join("")});g.module("ngSanitize").directive("ngBindHtml",["$sanitize",function(a){return function(d,b,e){b.addClass("ng-binding").data("$binding",e.ngBindHtml);d.$watch(e.ngBindHtml,function(c){c=a(c);b.html(c||"")})}}]);g.module("ngSanitize").filter("linky",function(){var a=/((ftp|https?):\/\/|(mailto:)?[A-Za-z0-9._%+-]+@)\S*[^\s\.\;\,\(\)\{\}\<\>]/,d=/^mailto:/;return function(b){if(!b)return b;for(var e=b,c=[],h=u(c),f,g;b=e.match(a);)f=b[0],b[2]==b[3]&&(f="mailto:"+f),g=b.index, 13 | h.chars(e.substr(0,g)),h.start("a",{href:f}),h.chars(b[0].replace(d,"")),h.end("a"),e=e.substring(g+b[0].length);h.chars(e);return c.join("")}})})(window,window.angular); 14 | -------------------------------------------------------------------------------- /website/import_bano.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import csv 4 | import sys 5 | import os 6 | import json 7 | from elasticsearch import Elasticsearch 8 | from elasticsearch.helpers import bulk_index 9 | 10 | ES = Elasticsearch() 11 | INDEX = 'photon' 12 | DOC_TYPE = 'place' 13 | FILEPATH = os.environ.get('BANO_FILEPATH', 'bano.csv') 14 | DUMPPATH = os.environ.get('BANO_DUMPPATH', '/tmp') 15 | 16 | fields = [ 17 | 'source_id', 'housenumber', 'street', 'postcode', 'city', 'source', 'lat', 18 | 'lon', 'dep', 'region' 19 | ] 20 | 21 | 22 | def row_to_doc(row): 23 | context = ', '.join([row['dep'], row['region']]) 24 | return { 25 | "osm_id": 123456789, # Fix me when we have a source_id 26 | "osm_type": "N", 27 | "osm_key": "place", 28 | "osm_value": "house", 29 | "importance": 0.0, 30 | "coordinate": { 31 | "lat": row['lat'], 32 | "lon": row['lon'] 33 | }, 34 | "housenumber": row['housenumber'], 35 | "postcode": row['postcode'], 36 | "city": { 37 | "default": row['city'], 38 | "fr": row['city'] 39 | }, 40 | "country": { 41 | "de": "Frankreich", 42 | "it": "Francia", 43 | "default": "France", 44 | "fr": "France", 45 | "en": "France" 46 | }, 47 | "street": { 48 | "default": row['street'], 49 | }, 50 | "context": { 51 | "default": context, 52 | "fr": context, 53 | } 54 | } 55 | 56 | 57 | def cleanup(): 58 | query = { 59 | "query": { 60 | "filtered": { 61 | "query": { 62 | "match_all": {} 63 | }, 64 | "filter": { 65 | "and": { 66 | "filters": [ 67 | { 68 | "exists": { 69 | "field": "housenumber" 70 | } 71 | }, 72 | { 73 | "term": { 74 | "osm_value": "house" 75 | } 76 | } 77 | ] 78 | } 79 | } 80 | } 81 | } 82 | } 83 | ES.delete_by_query( 84 | index=INDEX, 85 | doc_type=DOC_TYPE, 86 | body=query 87 | ) 88 | 89 | 90 | def index(data): 91 | print('Start indexing batch of', len(data)) 92 | bulk_index(ES, data, index=INDEX, doc_type=DOC_TYPE, refresh=True) 93 | print('End indexing of current batch') 94 | 95 | 96 | def dump(data, idx): 97 | path = os.path.join( 98 | DUMPPATH, 99 | 'bano_dump_{}'.format(idx) 100 | ) 101 | with open(path, mode='w', encoding='utf-8') as f: 102 | f.write('\n'.join(data)) 103 | sys.stdout.write('Dump {0}\n'.format(path)) 104 | 105 | 106 | if __name__ == "__main__": 107 | # first cleanup the housenumber data (we don't want duplicates) 108 | cleanup() 109 | with open(FILEPATH) as f: 110 | reader = csv.DictReader(f, fieldnames=fields) 111 | count = 0 112 | data = [] 113 | idx = 0 114 | for row in reader: 115 | data.append('{"index": {}}') 116 | data.append(json.dumps(row_to_doc(row))) 117 | count += 1 118 | if count % 100000 == 0: 119 | dump(data, idx) 120 | idx += 1 121 | data = [] 122 | sys.stdout.write("Done {}\n".format(count)) 123 | if data: 124 | dump(data, idx) 125 | sys.stdout.write("Done {}\n".format(count)) 126 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/nominatim/ImportThread.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.nominatim; 2 | 3 | import de.komoot.photon.Importer; 4 | import de.komoot.photon.PhotonDoc; 5 | import org.apache.logging.log4j.LogManager; 6 | import org.apache.logging.log4j.Logger; 7 | 8 | import java.util.List; 9 | import java.util.concurrent.BlockingQueue; 10 | import java.util.concurrent.LinkedBlockingDeque; 11 | import java.util.concurrent.atomic.AtomicLong; 12 | 13 | /** 14 | * Worker thread for bulk importing data from a Nominatim database. 15 | */ 16 | public class ImportThread { 17 | private static final Logger LOGGER = LogManager.getLogger(); 18 | 19 | private static final int PROGRESS_INTERVAL = 50000; 20 | private static final List FINAL_DOCUMENT = List.of(); 21 | private final BlockingQueue> documents = new LinkedBlockingDeque<>(100); 22 | private final AtomicLong counter = new AtomicLong(); 23 | private final Importer importer; 24 | private final Thread thread; 25 | private final long startMillis; 26 | 27 | public ImportThread(Importer importer) { 28 | this.importer = importer; 29 | this.thread = new Thread(new ImportRunnable()); 30 | this.thread.start(); 31 | this.startMillis = System.currentTimeMillis(); 32 | } 33 | 34 | /** 35 | * Adds the given document from Nominatim to the import queue. 36 | * 37 | * @param docs Fully filled nominatim document. 38 | */ 39 | public void addDocument(Iterable docs) { 40 | if (docs == null || !docs.iterator().hasNext()) { 41 | return; 42 | } 43 | 44 | while (true) { 45 | try { 46 | documents.put(docs); 47 | break; 48 | } catch (InterruptedException e) { 49 | LOGGER.warn("Thread interrupted while placing document in queue."); 50 | // Restore interrupted state. 51 | Thread.currentThread().interrupt(); 52 | } 53 | } 54 | 55 | if (counter.incrementAndGet() % PROGRESS_INTERVAL == 0) { 56 | final double documentsPerSecond = 1000d * counter.longValue() / (System.currentTimeMillis() - startMillis); 57 | LOGGER.info("Imported {} documents [{}/second]", counter.longValue(), documentsPerSecond); 58 | } 59 | } 60 | 61 | /** 62 | * Finalize the import. 63 | * Sends an end marker to the import thread and then waits for it to join. 64 | */ 65 | public void finish() { 66 | while (true) { 67 | try { 68 | documents.put(FINAL_DOCUMENT); 69 | thread.join(); 70 | break; 71 | } catch (InterruptedException e) { 72 | LOGGER.warn("Thread interrupted while placing document in queue."); 73 | // Restore interrupted state. 74 | Thread.currentThread().interrupt(); 75 | } 76 | } 77 | LOGGER.info("Finished import of {} photon documents. (Total processing time: {}s)", 78 | counter.longValue(), (System.currentTimeMillis() - startMillis)/1000); 79 | } 80 | 81 | private class ImportRunnable implements Runnable { 82 | 83 | @Override 84 | public void run() { 85 | while (true) { 86 | try { 87 | final var docs = documents.take(); 88 | if (!docs.iterator().hasNext()) { 89 | break; 90 | } 91 | importer.add(docs); 92 | } catch (InterruptedException e) { 93 | LOGGER.info("Interrupted exception", e); 94 | // Restore interrupted state. 95 | Thread.currentThread().interrupt(); 96 | } 97 | } 98 | importer.finish(); 99 | } 100 | } 101 | 102 | } 103 | -------------------------------------------------------------------------------- /src/test/java/de/komoot/photon/TestServer.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon; 2 | 3 | import de.komoot.photon.opensearch.OpenSearchResult; 4 | import de.komoot.photon.opensearch.PhotonIndex; 5 | import de.komoot.photon.searcher.PhotonResult; 6 | import org.codelibs.opensearch.runner.OpenSearchRunner; 7 | import org.opensearch.common.settings.Settings; 8 | 9 | import java.io.IOException; 10 | import java.util.List; 11 | import java.util.stream.Collectors; 12 | 13 | public class TestServer extends Server { 14 | public static final String TEST_CLUSTER_NAME = "photon-test"; 15 | 16 | private OpenSearchRunner runner; 17 | private String instanceDir; 18 | 19 | public TestServer(String mainDirectory) { 20 | super(mainDirectory); 21 | 22 | instanceDir = mainDirectory; 23 | } 24 | 25 | public void startTestServer(String clusterName) throws IOException { 26 | runner = new OpenSearchRunner(); 27 | runner.onBuild(new OpenSearchRunner.Builder() { 28 | @Override 29 | public void build(final int number, final Settings.Builder settingsBuilder) { 30 | settingsBuilder.put("http.cors.enabled", true); 31 | settingsBuilder.put("http.cors.allow-origin", "*"); 32 | settingsBuilder.put("discovery.type", "single-node"); 33 | settingsBuilder.putList("discovery.seed_hosts", "127.0.0.1:9201"); 34 | settingsBuilder.put("logger.org.opensearch.cluster.metadata", "TRACE"); 35 | settingsBuilder.put("cluster.search.request.slowlog.level", "TRACE"); 36 | settingsBuilder.put("cluster.search.request.slowlog.threshold.warn", "0ms"); 37 | settingsBuilder.put("cluster.search.request.slowlog.threshold.info", "0ms"); 38 | settingsBuilder.put("cluster.search.request.slowlog.threshold.debug", "0ms"); 39 | settingsBuilder.put("cluster.search.request.slowlog.threshold.trace", "0ms"); 40 | 41 | } 42 | }).build(OpenSearchRunner.newConfigs() 43 | .basePath(instanceDir) 44 | .clusterName(clusterName) 45 | .numOfNode(1) 46 | .baseHttpPort(9200)); 47 | 48 | // wait for yellow status 49 | runner.ensureYellow(); 50 | 51 | String[] transportAddresses = {"127.0.0.1:" + runner.node().settings().get("http.port")}; 52 | start(clusterName, transportAddresses, true); 53 | } 54 | 55 | public void stopTestServer() { 56 | shutdown(); 57 | try { 58 | runner.close(); 59 | } catch (IOException e) { 60 | throw new RuntimeException(e); 61 | } 62 | runner.clean(); 63 | } 64 | 65 | public String getHttpPort() { 66 | return runner.node().settings().get("http.port"); 67 | } 68 | 69 | public void refreshTestServer() { 70 | try { 71 | refreshIndexes(); 72 | } catch (IOException e) { 73 | throw new RuntimeException(e); 74 | } 75 | } 76 | 77 | public PhotonResult getByID(String id) { 78 | try { 79 | final var response = client.get(fn -> fn 80 | .index(PhotonIndex.NAME) 81 | .id(id), OpenSearchResult.class); 82 | 83 | if (response.found()) { 84 | return response.source(); 85 | } 86 | } catch (IOException e) { 87 | // ignore 88 | } 89 | 90 | return null; 91 | } 92 | 93 | public List getAll() { 94 | try { 95 | final var response = client.search(s -> s.size(1000), OpenSearchResult.class); 96 | 97 | return response.hits().hits() 98 | .stream().map(h -> h.source()) 99 | .collect(Collectors.toList()); 100 | } catch (IOException e) { 101 | //ignore 102 | } 103 | 104 | return List.of(); 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/opensearch/OpenSearchStructuredSearchHandler.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.opensearch; 2 | 3 | import de.komoot.photon.searcher.PhotonResult; 4 | import de.komoot.photon.query.StructuredSearchRequest; 5 | import de.komoot.photon.searcher.SearchHandler; 6 | import org.opensearch.client.opensearch.OpenSearchClient; 7 | import org.opensearch.client.opensearch._types.SearchType; 8 | import org.opensearch.client.opensearch._types.query_dsl.Query; 9 | import org.opensearch.client.opensearch.core.SearchResponse; 10 | 11 | import java.util.ArrayList; 12 | import java.util.List; 13 | import java.io.IOException; 14 | 15 | /** 16 | * Execute a structured forward lookup on an Elasticsearch database. 17 | */ 18 | public class OpenSearchStructuredSearchHandler implements SearchHandler { 19 | private final OpenSearchClient client; 20 | private final String queryTimeout; 21 | 22 | public OpenSearchStructuredSearchHandler(OpenSearchClient client, int queryTimeoutSec) { 23 | this.client = client; 24 | queryTimeout = queryTimeoutSec + "s"; 25 | } 26 | 27 | @Override 28 | public List search(StructuredSearchRequest photonRequest) { 29 | // for the case of deduplication we need a bit more results, #300 30 | int limit = photonRequest.getLimit(); 31 | int extLimit = limit > 1 ? (int) Math.round(photonRequest.getLimit() * 1.5) : 1; 32 | 33 | var results = sendQuery(buildQuery(photonRequest, false), extLimit); 34 | 35 | if (results.hits().total() != null && results.hits().total().value() == 0) { 36 | results = sendQuery(buildQuery(photonRequest, true), extLimit); 37 | 38 | if (results.hits().total() != null && results.hits().total().value() == 0 && photonRequest.hasStreet()) { 39 | var street = photonRequest.getStreet(); 40 | var houseNumber = photonRequest.getHouseNumber(); 41 | photonRequest.setStreet(null); 42 | photonRequest.setHouseNumber(null); 43 | results = sendQuery(buildQuery(photonRequest, true), extLimit); 44 | photonRequest.setStreet(street); 45 | photonRequest.setHouseNumber(houseNumber); 46 | } 47 | } 48 | 49 | List ret = new ArrayList<>(); 50 | for (var hit : results.hits().hits()) { 51 | var source = hit.source(); 52 | var score = hit.score(); 53 | if (source != null) { 54 | if (score != null) { 55 | source.setScore(score); 56 | } 57 | ret.add(source); 58 | } 59 | } 60 | 61 | return ret; 62 | } 63 | 64 | @Override 65 | public String dumpQuery(StructuredSearchRequest searchRequest) { 66 | return "{}"; 67 | } 68 | 69 | public Query buildQuery(StructuredSearchRequest photonRequest, boolean lenient) { 70 | final var query = new SearchQueryBuilder(photonRequest, lenient); 71 | query.addOsmTagFilter(photonRequest.getOsmTagFilters()); 72 | query.addLayerFilter(photonRequest.getLayerFilters()); 73 | query.addLocationBias(photonRequest.getLocationForBias(), photonRequest.getScaleForBias(), photonRequest.getZoomForBias()); 74 | query.includeCategories(photonRequest.getIncludeCategories()); 75 | query.excludeCategories(photonRequest.getExcludeCategories()); 76 | query.addBoundingBox(photonRequest.getBbox()); 77 | 78 | return query.build(); 79 | } 80 | 81 | private SearchResponse sendQuery(Query query, Integer limit) { 82 | try { 83 | return client.search(s -> s 84 | .index(PhotonIndex.NAME) 85 | .searchType(SearchType.QueryThenFetch) 86 | .query(query) 87 | .size(limit) 88 | .timeout(queryTimeout), OpenSearchResult.class); 89 | } catch (IOException e) { 90 | throw new RuntimeException("IO error during search", e); 91 | } 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /src/test/java/de/komoot/photon/query/QueryByLanguageTest.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.query; 2 | 3 | import de.komoot.photon.ESBaseTester; 4 | import de.komoot.photon.PhotonDoc; 5 | import de.komoot.photon.Importer; 6 | import de.komoot.photon.nominatim.model.AddressType; 7 | import de.komoot.photon.searcher.PhotonResult; 8 | import org.junit.jupiter.api.Test; 9 | import org.junit.jupiter.api.io.TempDir; 10 | import org.junit.jupiter.params.ParameterizedTest; 11 | import org.junit.jupiter.params.provider.EnumSource; 12 | import org.junit.jupiter.params.provider.ValueSource; 13 | 14 | import static org.assertj.core.api.Assertions.*; 15 | 16 | import java.io.IOException; 17 | import java.nio.file.Path; 18 | import java.util.*; 19 | 20 | /** 21 | * Tests for queries in different languages. 22 | */ 23 | class QueryByLanguageTest extends ESBaseTester { 24 | private int testDocId = 10001; 25 | private String[] languageList; 26 | 27 | @TempDir 28 | private Path dataDirectory; 29 | 30 | private Importer setup(String... languages) throws IOException { 31 | languageList = languages; 32 | getProperties().setLanguages(languages); 33 | setUpES(dataDirectory); 34 | return makeImporter(); 35 | } 36 | 37 | private PhotonDoc createDoc(String... names) { 38 | ++testDocId; 39 | return new PhotonDoc() 40 | .placeId(testDocId).osmType("W").osmId(testDocId).tagKey("place").tagValue("city") 41 | .names(makeDocNames(names)); 42 | } 43 | 44 | private List search(String query, String lang) { 45 | final var request = new SimpleSearchRequest(); 46 | request.setQuery(query); 47 | request.setLanguage(lang); 48 | 49 | return getServer().createSearchHandler(1).search(request); 50 | } 51 | 52 | @Test 53 | void queryNonStandardLanguages() throws IOException { 54 | Importer instance = setup("en", "fi"); 55 | 56 | instance.add(List.of( 57 | createDoc("name", "original", "name:fi", "finish", "name:ru", "russian"))); 58 | 59 | instance.finish(); 60 | refresh(); 61 | 62 | assertThat(search("original", "en")).hasSize(1); 63 | assertThat(search("finish", "en")).hasSize(1); 64 | assertThat(search("russian", "en")).hasSize(0); 65 | } 66 | 67 | @Test 68 | void queryAltNames() throws IOException { 69 | Importer instance = setup("de"); 70 | instance.add(List.of( 71 | createDoc("name", "simple", "alt_name", "ancient", "name:de", "einfach"))); 72 | instance.finish(); 73 | refresh(); 74 | 75 | assertThat(search("simple", "de")).hasSize(1); 76 | assertThat(search("einfach", "de")).hasSize(1); 77 | assertThat(search("ancient", "de")).hasSize(1); 78 | } 79 | 80 | @ParameterizedTest 81 | @EnumSource(names = {"STREET", "LOCALITY", "DISTRICT", "CITY", "COUNTRY", "STATE"}) 82 | void queryAddressPartsLanguages(AddressType addressType) throws IOException { 83 | Importer instance = setup("en", "de"); 84 | 85 | PhotonDoc doc = createDoc("name", "here").tagKey("place").tagValue("house"); 86 | 87 | doc.setAddressPartIfNew(addressType, makeAddressNames( 88 | "name", "original", 89 | "name:de", "deutsch")); 90 | 91 | instance.add(List.of(doc)); 92 | instance.finish(); 93 | refresh(); 94 | 95 | assertThat(search("here, original", "de")).hasSize(1); 96 | assertThat(search("here, Deutsch", "de")).hasSize(1); 97 | } 98 | 99 | @ParameterizedTest 100 | @ValueSource(strings = {"default", "de", "en"}) 101 | void queryAltNamesFuzzy(String lang) throws IOException { 102 | Importer instance = setup("de", "en"); 103 | instance.add(List.of( 104 | createDoc("name", "simple", "alt_name", "ancient", "name:de", "einfach"))); 105 | instance.finish(); 106 | refresh(); 107 | 108 | assertThat(search("simplle", lang)).hasSize(1); 109 | assertThat(search("einfah", lang)).hasSize(1); 110 | assertThat(search("anciemt", lang)).hasSize(1); 111 | assertThat(search("sinister", lang)).hasSize(0); 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /src/main/java/de/komoot/photon/searcher/GeocodeJsonFormatter.java: -------------------------------------------------------------------------------- 1 | package de.komoot.photon.searcher; 2 | 3 | import com.fasterxml.jackson.core.JsonGenerator; 4 | import com.fasterxml.jackson.core.JsonProcessingException; 5 | import com.fasterxml.jackson.databind.ObjectMapper; 6 | import de.komoot.photon.Constants; 7 | 8 | import java.io.IOException; 9 | import java.io.StringWriter; 10 | import java.util.List; 11 | import java.util.Map; 12 | 13 | /** 14 | * Format a database result into a Photon GeocodeJson response. 15 | */ 16 | public class GeocodeJsonFormatter implements ResultFormatter { 17 | private static final String[] KEYS_LANG_UNSPEC = {Constants.OSM_TYPE, Constants.OSM_ID, Constants.OSM_KEY, Constants.OSM_VALUE, Constants.OBJECT_TYPE, Constants.POSTCODE, Constants.HOUSENUMBER, Constants.COUNTRYCODE}; 18 | private static final String[] KEYS_LANG_SPEC = {Constants.NAME, Constants.COUNTRY, Constants.CITY, Constants.DISTRICT, Constants.LOCALITY, Constants.STREET, Constants.STATE, Constants.COUNTY}; 19 | 20 | private final ObjectMapper mapper = new ObjectMapper(); 21 | 22 | @Override 23 | public String formatError(String msg) { 24 | try { 25 | return mapper.writeValueAsString( 26 | Map.of("message", msg == null ? "Unknown error." : msg)); 27 | } catch (JsonProcessingException e) { 28 | return "{}"; 29 | } 30 | } 31 | 32 | @Override 33 | public String convert(List results, String language, 34 | boolean withGeometry, boolean withDebugInfo, String queryDebugInfo) throws IOException { 35 | final var writer = new StringWriter(); 36 | 37 | try (var gen = mapper.createGenerator(writer)) { 38 | if (withDebugInfo) { 39 | gen.useDefaultPrettyPrinter(); 40 | } 41 | 42 | gen.writeStartObject(); 43 | gen.writeStringField("type", "FeatureCollection"); 44 | gen.writeArrayFieldStart("features"); 45 | 46 | for (PhotonResult result : results) { 47 | gen.writeStartObject(); 48 | gen.writeStringField("type", "Feature"); 49 | 50 | gen.writeObjectFieldStart("properties"); 51 | 52 | for (String key : KEYS_LANG_UNSPEC) { 53 | put(gen, key, result.get(key)); 54 | } 55 | 56 | for (String key : KEYS_LANG_SPEC) { 57 | put(gen, key, result.getLocalised(key, language)); 58 | } 59 | 60 | put(gen, "extent", result.getExtent()); 61 | 62 | put(gen, "extra", result.getMap("extra")); 63 | 64 | gen.writeEndObject(); 65 | 66 | if (withGeometry && result.getGeometry() != null) { 67 | gen.writeFieldName("geometry"); 68 | gen.writeRawValue(result.getGeometry()); 69 | } else { 70 | gen.writeObjectFieldStart("geometry"); 71 | gen.writeStringField("type", "Point"); 72 | gen.writeObjectField("coordinates", result.getCoordinates()); 73 | gen.writeEndObject(); 74 | } 75 | 76 | gen.writeEndObject(); 77 | } 78 | 79 | gen.writeEndArray(); 80 | 81 | if (withDebugInfo || queryDebugInfo != null) { 82 | gen.writeObjectFieldStart("properties"); 83 | if (queryDebugInfo != null) { 84 | gen.writeFieldName("debug"); 85 | gen.writeRawValue(queryDebugInfo); 86 | } 87 | if (withDebugInfo) { 88 | gen.writeArrayFieldStart("raw_data"); 89 | for (var res : results) { 90 | gen.writePOJO(res.getRawData()); 91 | } 92 | gen.writeEndArray(); 93 | } 94 | gen.writeEndObject(); 95 | } 96 | 97 | gen.writeEndObject(); 98 | } 99 | 100 | return writer.toString(); 101 | } 102 | 103 | 104 | private void put(JsonGenerator gen, String key, Object value) throws IOException { 105 | if (value != null) { 106 | gen.writeObjectField(key, value); 107 | } 108 | } 109 | } 110 | --------------------------------------------------------------------------------